2024-11-16 05:39:29,540 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 05:39:29,559 main DEBUG Took 0.016088 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 05:39:29,560 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 05:39:29,560 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 05:39:29,562 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 05:39:29,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,575 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 05:39:29,599 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,601 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,602 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,603 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,604 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,604 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,606 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,606 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,607 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,608 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,609 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,609 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,610 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,610 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,611 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,612 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,612 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,613 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,616 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,617 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,619 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,620 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:39:29,622 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,622 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 05:39:29,624 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:39:29,626 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 05:39:29,629 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 05:39:29,630 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 05:39:29,634 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 05:39:29,635 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 05:39:29,647 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 05:39:29,651 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 05:39:29,655 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 05:39:29,656 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 05:39:29,657 main DEBUG createAppenders(={Console}) 2024-11-16 05:39:29,661 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-16 05:39:29,663 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 05:39:29,664 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-16 05:39:29,665 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 05:39:29,666 main DEBUG OutputStream closed 2024-11-16 05:39:29,668 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 05:39:29,669 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 05:39:29,669 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-16 05:39:29,811 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 05:39:29,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 05:39:29,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 05:39:29,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 05:39:29,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 05:39:29,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 05:39:29,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 05:39:29,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 05:39:29,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 05:39:29,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 05:39:29,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 05:39:29,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 05:39:29,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 05:39:29,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 05:39:29,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 05:39:29,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 05:39:29,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 05:39:29,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 05:39:29,832 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 05:39:29,833 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-16 05:39:29,833 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 05:39:29,834 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-16T05:39:30,181 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466 2024-11-16 05:39:30,185 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 05:39:30,185 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T05:39:30,198 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-16T05:39:30,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T05:39:30,252 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/cluster_d224ca95-f133-78db-847d-de6e87cd9c78, deleteOnExit=true 2024-11-16T05:39:30,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T05:39:30,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/test.cache.data in system properties and HBase conf 2024-11-16T05:39:30,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T05:39:30,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/hadoop.log.dir in system properties and HBase conf 2024-11-16T05:39:30,257 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T05:39:30,258 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T05:39:30,259 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T05:39:30,390 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T05:39:30,542 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:39:30,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T05:39:30,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T05:39:30,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T05:39:30,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T05:39:30,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T05:39:30,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T05:39:30,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T05:39:30,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T05:39:30,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T05:39:30,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/nfs.dump.dir in system properties and HBase conf 2024-11-16T05:39:30,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/java.io.tmpdir in system properties and HBase conf 2024-11-16T05:39:30,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T05:39:30,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T05:39:30,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T05:39:31,567 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T05:39:31,675 INFO [Time-limited test {}] log.Log(170): Logging initialized @3041ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T05:39:31,760 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:39:31,836 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:39:31,857 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:39:31,858 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:39:31,859 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T05:39:31,874 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:39:31,878 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:39:31,879 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:39:32,087 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/java.io.tmpdir/jetty-localhost-40999-hadoop-hdfs-3_4_1-tests_jar-_-any-3695782818408651507/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T05:39:32,096 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40999} 2024-11-16T05:39:32,096 INFO [Time-limited test {}] server.Server(415): Started @3462ms 2024-11-16T05:39:32,514 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:39:32,522 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:39:32,524 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:39:32,524 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:39:32,524 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T05:39:32,525 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:39:32,525 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:39:32,628 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/java.io.tmpdir/jetty-localhost-40647-hadoop-hdfs-3_4_1-tests_jar-_-any-2761282464497640720/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:39:32,629 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40647} 2024-11-16T05:39:32,629 INFO [Time-limited test {}] server.Server(415): Started @3995ms 2024-11-16T05:39:32,682 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T05:39:33,324 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/cluster_d224ca95-f133-78db-847d-de6e87cd9c78/data/data1/current/BP-2113210957-172.17.0.2-1731735571282/current, will proceed with Du for space computation calculation, 2024-11-16T05:39:33,324 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/cluster_d224ca95-f133-78db-847d-de6e87cd9c78/data/data2/current/BP-2113210957-172.17.0.2-1731735571282/current, will proceed with Du for space computation calculation, 2024-11-16T05:39:33,453 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T05:39:33,541 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb0e8fa7c1ad883ea with lease ID 0x9cb2bbfdefa64c6c: Processing first storage report for DS-c54bf3bd-6f78-4228-9fd7-5cc9c448a296 from datanode DatanodeRegistration(127.0.0.1:33791, datanodeUuid=122d7910-34dd-4638-982b-e246163848a6, infoPort=40143, infoSecurePort=0, ipcPort=44599, storageInfo=lv=-57;cid=testClusterID;nsid=1930664696;c=1731735571282) 2024-11-16T05:39:33,544 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb0e8fa7c1ad883ea with lease ID 0x9cb2bbfdefa64c6c: from storage DS-c54bf3bd-6f78-4228-9fd7-5cc9c448a296 node DatanodeRegistration(127.0.0.1:33791, datanodeUuid=122d7910-34dd-4638-982b-e246163848a6, infoPort=40143, infoSecurePort=0, ipcPort=44599, storageInfo=lv=-57;cid=testClusterID;nsid=1930664696;c=1731735571282), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-16T05:39:33,544 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb0e8fa7c1ad883ea with lease ID 0x9cb2bbfdefa64c6c: Processing first storage report for DS-27ac9f20-080a-40de-b878-ab898f0b9cf4 from datanode DatanodeRegistration(127.0.0.1:33791, datanodeUuid=122d7910-34dd-4638-982b-e246163848a6, infoPort=40143, infoSecurePort=0, ipcPort=44599, storageInfo=lv=-57;cid=testClusterID;nsid=1930664696;c=1731735571282) 2024-11-16T05:39:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb0e8fa7c1ad883ea with lease ID 0x9cb2bbfdefa64c6c: from storage DS-27ac9f20-080a-40de-b878-ab898f0b9cf4 node DatanodeRegistration(127.0.0.1:33791, datanodeUuid=122d7910-34dd-4638-982b-e246163848a6, infoPort=40143, infoSecurePort=0, ipcPort=44599, storageInfo=lv=-57;cid=testClusterID;nsid=1930664696;c=1731735571282), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T05:39:33,546 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466 2024-11-16T05:39:33,645 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/cluster_d224ca95-f133-78db-847d-de6e87cd9c78/zookeeper_0, clientPort=53518, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/cluster_d224ca95-f133-78db-847d-de6e87cd9c78/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/cluster_d224ca95-f133-78db-847d-de6e87cd9c78/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T05:39:33,657 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53518 2024-11-16T05:39:33,670 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:39:33,673 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:39:33,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741825_1001 (size=7) 2024-11-16T05:39:34,378 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d with version=8 2024-11-16T05:39:34,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/hbase-staging 2024-11-16T05:39:34,486 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T05:39:34,713 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:39:34,722 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:39:34,722 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:39:34,729 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:39:34,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:39:34,730 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:39:34,886 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T05:39:34,953 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T05:39:34,966 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T05:39:34,971 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:39:35,005 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 991 (auto-detected) 2024-11-16T05:39:35,007 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T05:39:35,029 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39409 2024-11-16T05:39:35,051 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39409 connecting to ZooKeeper ensemble=127.0.0.1:53518 2024-11-16T05:39:35,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:394090x0, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:39:35,079 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39409-0x100470b30e60000 connected 2024-11-16T05:39:35,104 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:39:35,106 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:39:35,115 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:39:35,119 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d, hbase.cluster.distributed=false 2024-11-16T05:39:35,147 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:39:35,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39409 2024-11-16T05:39:35,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39409 2024-11-16T05:39:35,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39409 2024-11-16T05:39:35,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39409 2024-11-16T05:39:35,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39409 2024-11-16T05:39:35,252 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:39:35,253 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:39:35,254 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:39:35,254 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:39:35,254 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:39:35,254 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:39:35,257 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:39:35,259 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:39:35,260 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35555 2024-11-16T05:39:35,262 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35555 connecting to ZooKeeper ensemble=127.0.0.1:53518 2024-11-16T05:39:35,264 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:39:35,266 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:39:35,273 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355550x0, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:39:35,274 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35555-0x100470b30e60001 connected 2024-11-16T05:39:35,274 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:39:35,277 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:39:35,285 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:39:35,287 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:39:35,293 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:39:35,294 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35555 2024-11-16T05:39:35,294 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35555 2024-11-16T05:39:35,295 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35555 2024-11-16T05:39:35,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35555 2024-11-16T05:39:35,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35555 2024-11-16T05:39:35,317 DEBUG [M:0;3456ee6a3164:39409 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3456ee6a3164:39409 2024-11-16T05:39:35,318 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3456ee6a3164,39409,1731735574536 2024-11-16T05:39:35,325 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:39:35,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:39:35,327 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3456ee6a3164,39409,1731735574536 2024-11-16T05:39:35,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:35,349 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:39:35,350 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:35,350 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T05:39:35,351 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3456ee6a3164,39409,1731735574536 from backup master directory 2024-11-16T05:39:35,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3456ee6a3164,39409,1731735574536 2024-11-16T05:39:35,355 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:39:35,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:39:35,356 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:39:35,356 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3456ee6a3164,39409,1731735574536 2024-11-16T05:39:35,358 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T05:39:35,359 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T05:39:35,415 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/hbase.id] with ID: c99181d9-1a27-4d42-b12d-1ee1970e5198 2024-11-16T05:39:35,416 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/.tmp/hbase.id 2024-11-16T05:39:35,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741826_1002 (size=42) 2024-11-16T05:39:35,830 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/.tmp/hbase.id]:[hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/hbase.id] 2024-11-16T05:39:35,875 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:39:35,880 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T05:39:35,904 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-16T05:39:35,908 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:35,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:35,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741827_1003 (size=196) 2024-11-16T05:39:36,344 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:39:36,347 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T05:39:36,365 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:36,369 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:39:36,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741828_1004 (size=1189) 2024-11-16T05:39:36,822 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store 2024-11-16T05:39:36,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741829_1005 (size=34) 2024-11-16T05:39:37,253 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T05:39:37,257 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:39:37,259 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:39:37,259 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:39:37,259 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:39:37,261 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:39:37,262 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:39:37,262 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:39:37,263 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731735577258Disabling compacts and flushes for region at 1731735577258Disabling writes for close at 1731735577262 (+4 ms)Writing region close event to WAL at 1731735577262Closed at 1731735577262 2024-11-16T05:39:37,266 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/.initializing 2024-11-16T05:39:37,266 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/WALs/3456ee6a3164,39409,1731735574536 2024-11-16T05:39:37,277 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:39:37,298 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C39409%2C1731735574536, suffix=, logDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/WALs/3456ee6a3164,39409,1731735574536, archiveDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/oldWALs, maxLogs=10 2024-11-16T05:39:37,322 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/WALs/3456ee6a3164,39409,1731735574536/3456ee6a3164%2C39409%2C1731735574536.1731735577303, exclude list is [], retry=0 2024-11-16T05:39:37,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33791,DS-c54bf3bd-6f78-4228-9fd7-5cc9c448a296,DISK] 2024-11-16T05:39:37,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T05:39:37,381 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/WALs/3456ee6a3164,39409,1731735574536/3456ee6a3164%2C39409%2C1731735574536.1731735577303 2024-11-16T05:39:37,382 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40143:40143)] 2024-11-16T05:39:37,382 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:39:37,383 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:39:37,388 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,389 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,458 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T05:39:37,463 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:37,466 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:39:37,467 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T05:39:37,471 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:37,473 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:39:37,473 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T05:39:37,477 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:37,478 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:39:37,478 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,482 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T05:39:37,482 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:37,483 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:39:37,484 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,488 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,489 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,494 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,495 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,498 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:39:37,502 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:39:37,509 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:39:37,511 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69294764, jitterRate=0.03257244825363159}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:39:37,520 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731735577403Initializing all the Stores at 1731735577405 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731735577406 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735577407 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735577407Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735577407Cleaning up temporary data from old regions at 1731735577495 (+88 ms)Region opened successfully at 1731735577519 (+24 ms) 2024-11-16T05:39:37,521 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T05:39:37,558 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526e69b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:39:37,585 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T05:39:37,594 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T05:39:37,594 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T05:39:37,597 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:39:37,598 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T05:39:37,602 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T05:39:37,602 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T05:39:37,625 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T05:39:37,633 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T05:39:37,635 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T05:39:37,638 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T05:39:37,639 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T05:39:37,641 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T05:39:37,643 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T05:39:37,647 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T05:39:37,649 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T05:39:37,651 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T05:39:37,654 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T05:39:37,672 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T05:39:37,673 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T05:39:37,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:39:37,678 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:39:37,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:37,678 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:37,681 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3456ee6a3164,39409,1731735574536, sessionid=0x100470b30e60000, setting cluster-up flag (Was=false) 2024-11-16T05:39:37,693 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:37,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:37,698 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T05:39:37,700 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,39409,1731735574536 2024-11-16T05:39:37,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:37,705 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:37,709 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T05:39:37,711 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,39409,1731735574536 2024-11-16T05:39:37,716 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T05:39:37,779 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:39:37,789 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T05:39:37,795 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T05:39:37,801 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3456ee6a3164,39409,1731735574536 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T05:39:37,808 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(746): ClusterId : c99181d9-1a27-4d42-b12d-1ee1970e5198 2024-11-16T05:39:37,808 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:39:37,809 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:39:37,809 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:39:37,809 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:39:37,809 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3456ee6a3164:0, corePoolSize=10, maxPoolSize=10 2024-11-16T05:39:37,809 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,810 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:39:37,810 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,810 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:39:37,813 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731735607813 2024-11-16T05:39:37,815 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T05:39:37,815 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:39:37,815 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:39:37,816 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:39:37,816 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T05:39:37,816 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T05:39:37,819 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:39:37,820 DEBUG [RS:0;3456ee6a3164:35555 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f0ea1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:39:37,820 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T05:39:37,821 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T05:39:37,821 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T05:39:37,821 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T05:39:37,823 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:37,823 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T05:39:37,822 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:37,826 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T05:39:37,827 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T05:39:37,828 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T05:39:37,833 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T05:39:37,834 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T05:39:37,836 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731735577835,5,FailOnTimeoutGroup] 2024-11-16T05:39:37,836 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731735577836,5,FailOnTimeoutGroup] 2024-11-16T05:39:37,836 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3456ee6a3164:35555 2024-11-16T05:39:37,836 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:37,837 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T05:39:37,838 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:37,838 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:37,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741831_1007 (size=1321) 2024-11-16T05:39:37,840 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:39:37,840 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:39:37,840 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:39:37,840 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T05:39:37,841 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d 2024-11-16T05:39:37,843 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,39409,1731735574536 with port=35555, startcode=1731735575219 2024-11-16T05:39:37,856 DEBUG [RS:0;3456ee6a3164:35555 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:39:37,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741832_1008 (size=32) 2024-11-16T05:39:37,891 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33473, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:39:37,899 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39409 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,35555,1731735575219 2024-11-16T05:39:37,902 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39409 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:37,917 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d 2024-11-16T05:39:37,917 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35865 2024-11-16T05:39:37,917 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:39:37,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:39:37,925 DEBUG [RS:0;3456ee6a3164:35555 {}] zookeeper.ZKUtil(111): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,35555,1731735575219 2024-11-16T05:39:37,925 WARN [RS:0;3456ee6a3164:35555 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:39:37,926 INFO [RS:0;3456ee6a3164:35555 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:39:37,926 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/WALs/3456ee6a3164,35555,1731735575219 2024-11-16T05:39:37,931 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,35555,1731735575219] 2024-11-16T05:39:37,958 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:39:37,979 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:39:37,984 INFO [RS:0;3456ee6a3164:35555 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:39:37,984 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:37,986 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:39:37,994 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:39:37,996 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:37,996 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,997 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,997 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,997 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,997 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,998 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:39:37,998 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,998 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,998 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,998 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,998 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,999 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:39:37,999 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:39:37,999 DEBUG [RS:0;3456ee6a3164:35555 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:39:38,000 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,001 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,001 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,001 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,002 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,002 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,35555,1731735575219-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:39:38,024 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:39:38,026 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,35555,1731735575219-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,026 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,026 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.Replication(171): 3456ee6a3164,35555,1731735575219 started 2024-11-16T05:39:38,046 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,047 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,35555,1731735575219, RpcServer on 3456ee6a3164/172.17.0.2:35555, sessionid=0x100470b30e60001 2024-11-16T05:39:38,047 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:39:38,048 DEBUG [RS:0;3456ee6a3164:35555 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,35555,1731735575219 2024-11-16T05:39:38,048 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,35555,1731735575219' 2024-11-16T05:39:38,048 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:39:38,049 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:39:38,050 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:39:38,050 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:39:38,050 DEBUG [RS:0;3456ee6a3164:35555 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,35555,1731735575219 2024-11-16T05:39:38,050 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,35555,1731735575219' 2024-11-16T05:39:38,050 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:39:38,051 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:39:38,052 DEBUG [RS:0;3456ee6a3164:35555 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:39:38,052 INFO [RS:0;3456ee6a3164:35555 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:39:38,052 INFO [RS:0;3456ee6a3164:35555 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:39:38,159 INFO [RS:0;3456ee6a3164:35555 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:39:38,164 INFO [RS:0;3456ee6a3164:35555 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C35555%2C1731735575219, suffix=, logDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/WALs/3456ee6a3164,35555,1731735575219, archiveDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/oldWALs, maxLogs=32 2024-11-16T05:39:38,191 DEBUG [RS:0;3456ee6a3164:35555 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/WALs/3456ee6a3164,35555,1731735575219/3456ee6a3164%2C35555%2C1731735575219.1731735578170, exclude list is [], retry=0 2024-11-16T05:39:38,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33791,DS-c54bf3bd-6f78-4228-9fd7-5cc9c448a296,DISK] 2024-11-16T05:39:38,208 INFO [RS:0;3456ee6a3164:35555 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/WALs/3456ee6a3164,35555,1731735575219/3456ee6a3164%2C35555%2C1731735575219.1731735578170 2024-11-16T05:39:38,209 DEBUG [RS:0;3456ee6a3164:35555 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40143:40143)] 2024-11-16T05:39:38,263 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:39:38,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T05:39:38,269 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:39:38,269 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:38,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:39:38,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T05:39:38,274 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:39:38,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:38,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:39:38,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T05:39:38,279 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:39:38,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:38,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:39:38,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T05:39:38,284 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:39:38,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:38,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:39:38,286 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:39:38,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740 2024-11-16T05:39:38,289 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740 2024-11-16T05:39:38,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:39:38,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:39:38,294 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:39:38,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:39:38,302 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:39:38,303 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70483684, jitterRate=0.05028873682022095}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:39:38,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731735578263Initializing all the Stores at 1731735578265 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731735578265Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731735578265Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735578265Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731735578265Cleaning up temporary data from old regions at 1731735578293 (+28 ms)Region opened successfully at 1731735578306 (+13 ms) 2024-11-16T05:39:38,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:39:38,307 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:39:38,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:39:38,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:39:38,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:39:38,309 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:39:38,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731735578307Disabling compacts and flushes for region at 1731735578307Disabling writes for close at 1731735578307Writing region close event to WAL at 1731735578309 (+2 ms)Closed at 1731735578309 2024-11-16T05:39:38,314 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:39:38,314 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T05:39:38,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T05:39:38,329 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:39:38,332 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T05:39:38,487 DEBUG [3456ee6a3164:39409 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T05:39:38,500 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:38,505 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,35555,1731735575219, state=OPENING 2024-11-16T05:39:38,510 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T05:39:38,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:38,512 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:39:38,513 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:39:38,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:39:38,515 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:39:38,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:39:38,694 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:39:38,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42519, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:39:38,707 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T05:39:38,708 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:39:38,708 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T05:39:38,712 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C35555%2C1731735575219.meta, suffix=.meta, logDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/WALs/3456ee6a3164,35555,1731735575219, archiveDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/oldWALs, maxLogs=32 2024-11-16T05:39:38,726 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/WALs/3456ee6a3164,35555,1731735575219/3456ee6a3164%2C35555%2C1731735575219.meta.1731735578714.meta, exclude list is [], retry=0 2024-11-16T05:39:38,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33791,DS-c54bf3bd-6f78-4228-9fd7-5cc9c448a296,DISK] 2024-11-16T05:39:38,733 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/WALs/3456ee6a3164,35555,1731735575219/3456ee6a3164%2C35555%2C1731735575219.meta.1731735578714.meta 2024-11-16T05:39:38,734 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40143:40143)] 2024-11-16T05:39:38,734 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:39:38,736 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T05:39:38,738 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T05:39:38,742 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T05:39:38,747 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T05:39:38,747 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:39:38,748 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T05:39:38,748 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T05:39:38,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T05:39:38,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:39:38,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:38,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:39:38,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T05:39:38,755 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:39:38,755 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:38,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:39:38,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T05:39:38,758 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:39:38,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:38,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:39:38,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T05:39:38,760 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:39:38,760 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:38,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:39:38,761 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:39:38,762 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740 2024-11-16T05:39:38,765 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740 2024-11-16T05:39:38,767 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:39:38,768 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:39:38,768 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:39:38,771 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:39:38,773 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68056218, jitterRate=0.014116674661636353}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:39:38,773 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T05:39:38,775 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731735578748Writing region info on filesystem at 1731735578748Initializing all the Stores at 1731735578750 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731735578750Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731735578750Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735578750Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731735578750Cleaning up temporary data from old regions at 1731735578768 (+18 ms)Running coprocessor post-open hooks at 1731735578773 (+5 ms)Region opened successfully at 1731735578775 (+2 ms) 2024-11-16T05:39:38,782 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731735578686 2024-11-16T05:39:38,792 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T05:39:38,793 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T05:39:38,794 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:38,796 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,35555,1731735575219, state=OPEN 2024-11-16T05:39:38,801 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:39:38,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:39:38,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:39:38,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:39:38,802 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:38,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T05:39:38,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,35555,1731735575219 in 285 msec 2024-11-16T05:39:38,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T05:39:38,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 491 msec 2024-11-16T05:39:38,818 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:39:38,818 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T05:39:38,838 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:38,839 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:38,857 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:38,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59081, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:38,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1330 sec 2024-11-16T05:39:38,879 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731735578878, completionTime=-1 2024-11-16T05:39:38,881 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-16T05:39:38,881 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:39:38,905 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-16T05:39:38,905 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731735638905 2024-11-16T05:39:38,906 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731735698905 2024-11-16T05:39:38,906 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-16T05:39:38,907 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,39409,1731735574536-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,908 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,39409,1731735574536-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,908 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,39409,1731735574536-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,909 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3456ee6a3164:39409, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,909 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,910 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:38,916 DEBUG [master/3456ee6a3164:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T05:39:38,941 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.585sec 2024-11-16T05:39:38,942 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T05:39:38,943 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T05:39:38,944 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T05:39:38,945 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T05:39:38,945 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T05:39:38,946 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,39409,1731735574536-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:39:38,946 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,39409,1731735574536-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T05:39:38,955 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T05:39:38,956 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T05:39:38,956 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,39409,1731735574536-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:39:39,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:39,018 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T05:39:39,019 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T05:39:39,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:39,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:39,030 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:39,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ecb092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:39,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:39,041 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:39,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:39,045 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:39,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:39,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3456ee6a3164,39409,1731735574536 2024-11-16T05:39:39,085 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=4329 2024-11-16T05:39:39,093 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:39:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,39409,1731735574536 2024-11-16T05:39:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13ee3c32 2024-11-16T05:39:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:39:39,105 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:39:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:39:39,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:39:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:39:39,125 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:39:39,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-16T05:39:39,127 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:39,130 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:39:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:39:39,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741835_1011 (size=959) 2024-11-16T05:39:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:39:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:39:39,546 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d 2024-11-16T05:39:39,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741836_1012 (size=53) 2024-11-16T05:39:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:39:39,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:39:39,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3992ee2db1edacc4ba76f5d2411f31eb, disabling compactions & flushes 2024-11-16T05:39:39,960 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:39,960 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:39,960 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. after waiting 0 ms 2024-11-16T05:39:39,960 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:39,960 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:39,961 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3992ee2db1edacc4ba76f5d2411f31eb: Waiting for close lock at 1731735579959Disabling compacts and flushes for region at 1731735579959Disabling writes for close at 1731735579960 (+1 ms)Writing region close event to WAL at 1731735579960Closed at 1731735579960 2024-11-16T05:39:39,965 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:39:39,972 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731735579966"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731735579966"}]},"ts":"1731735579966"} 2024-11-16T05:39:39,977 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:39:39,979 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:39:39,982 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735579979"}]},"ts":"1731735579979"} 2024-11-16T05:39:39,986 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:39:39,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3992ee2db1edacc4ba76f5d2411f31eb, ASSIGN}] 2024-11-16T05:39:39,990 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3992ee2db1edacc4ba76f5d2411f31eb, ASSIGN 2024-11-16T05:39:39,992 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3992ee2db1edacc4ba76f5d2411f31eb, ASSIGN; state=OFFLINE, location=3456ee6a3164,35555,1731735575219; forceNewPlan=false, retain=false 2024-11-16T05:39:40,144 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3992ee2db1edacc4ba76f5d2411f31eb, regionState=OPENING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:40,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3992ee2db1edacc4ba76f5d2411f31eb, ASSIGN because future has completed 2024-11-16T05:39:40,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:39:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:39:40,314 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:40,314 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:39:40,315 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,315 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:39:40,315 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,315 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,319 INFO [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,322 INFO [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3992ee2db1edacc4ba76f5d2411f31eb columnFamilyName A 2024-11-16T05:39:40,322 DEBUG [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:40,323 INFO [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] regionserver.HStore(327): Store=3992ee2db1edacc4ba76f5d2411f31eb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:39:40,323 INFO [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,326 INFO [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3992ee2db1edacc4ba76f5d2411f31eb columnFamilyName B 2024-11-16T05:39:40,326 DEBUG [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:40,327 INFO [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] regionserver.HStore(327): Store=3992ee2db1edacc4ba76f5d2411f31eb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:39:40,327 INFO [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,329 INFO [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3992ee2db1edacc4ba76f5d2411f31eb columnFamilyName C 2024-11-16T05:39:40,330 DEBUG [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:39:40,330 INFO [StoreOpener-3992ee2db1edacc4ba76f5d2411f31eb-1 {}] regionserver.HStore(327): Store=3992ee2db1edacc4ba76f5d2411f31eb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:39:40,331 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,332 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,333 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,336 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,336 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,337 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:39:40,340 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,344 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:39:40,345 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3992ee2db1edacc4ba76f5d2411f31eb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69199787, jitterRate=0.031157180666923523}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:39:40,345 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:40,346 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3992ee2db1edacc4ba76f5d2411f31eb: Running coprocessor pre-open hook at 1731735580316Writing region info on filesystem at 1731735580316Initializing all the Stores at 1731735580318 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735580318Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735580318Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735580319 (+1 ms)Cleaning up temporary data from old regions at 1731735580336 (+17 ms)Running coprocessor post-open hooks at 1731735580345 (+9 ms)Region opened successfully at 1731735580346 (+1 ms) 2024-11-16T05:39:40,348 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., pid=6, masterSystemTime=1731735580306 2024-11-16T05:39:40,351 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:40,352 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:40,353 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3992ee2db1edacc4ba76f5d2411f31eb, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:40,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:39:40,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T05:39:40,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 in 208 msec 2024-11-16T05:39:40,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T05:39:40,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3992ee2db1edacc4ba76f5d2411f31eb, ASSIGN in 376 msec 2024-11-16T05:39:40,370 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:39:40,370 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735580370"}]},"ts":"1731735580370"} 2024-11-16T05:39:40,374 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:39:40,375 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:39:40,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2600 sec 2024-11-16T05:39:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:39:41,281 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:39:41,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:41,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:41,293 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ecac36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:41,297 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:41,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d14ea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,300 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:41,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:41,307 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:39:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,39409,1731735574536 2024-11-16T05:39:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21ac4f6f 2024-11-16T05:39:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:39:41,313 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:39:41,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:41,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:41,320 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:41,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a220135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:41,324 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:41,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff12ec3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:41,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:41,328 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fa5eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:41,331 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f897912, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:41,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:41,335 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:41,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a4abfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:41,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:41,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:41,339 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43659da1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:41,342 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:41,343 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180c72f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:41,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:41,347 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2572aff2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:41,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:41,350 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@831686b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:41,353 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43df1407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:41,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:41,357 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15646188, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:41,360 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:41,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40038a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:41,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:41,364 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:41,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5558ab63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:41,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:41,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:41,367 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc83151, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:41,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:41,389 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d54ea4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:41,392 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:41,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f8b85f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,395 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:39:41,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:39:41,396 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:39:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:39:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:39:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788043de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:39:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:39:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:39:41,400 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:39:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad487e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:39:41,407 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:41,407 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:41,407 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:41,410 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:41,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:41,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:41,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:41,415 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:41,415 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:41,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:41,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:41,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:39:41,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:41,428 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:39:41,429 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:41,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:39:41,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:39:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:39:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:39:41,437 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:39:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:39:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:39:41,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:41,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:41,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:39:41,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:39:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:39:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:39:41,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:39:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-16T05:39:41,459 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:39:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:39:41,462 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:39:41,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:39:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:41,519 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:41,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:39:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:39:41,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/ad9914c52f004ee88c414c128a537730 is 50, key is test_row_0/A:col10/1731735581515/Put/seqid=0 2024-11-16T05:39:41,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:39:41,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:41,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:41,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:41,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:41,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735641660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735641667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:41,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735641678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:41,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735641682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735641685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:41,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741837_1013 (size=12001) 2024-11-16T05:39:41,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/ad9914c52f004ee88c414c128a537730 2024-11-16T05:39:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:39:41,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/dc69df6d0d4044a0ac5e93a6780f864d is 50, key is test_row_0/B:col10/1731735581515/Put/seqid=0 2024-11-16T05:39:41,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:39:41,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:41,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:41,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:41,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:41,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:41,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735641831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:41,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741838_1014 (size=12001) 2024-11-16T05:39:41,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/dc69df6d0d4044a0ac5e93a6780f864d 2024-11-16T05:39:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735641834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:41,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735641834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:41,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735641834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:41,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735641835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:41,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/cb0e73e78f9142639be4dfe235692598 is 50, key is test_row_0/C:col10/1731735581515/Put/seqid=0 2024-11-16T05:39:41,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741839_1015 (size=12001) 2024-11-16T05:39:41,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/cb0e73e78f9142639be4dfe235692598 2024-11-16T05:39:41,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/ad9914c52f004ee88c414c128a537730 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ad9914c52f004ee88c414c128a537730 2024-11-16T05:39:41,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ad9914c52f004ee88c414c128a537730, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T05:39:41,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/dc69df6d0d4044a0ac5e93a6780f864d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/dc69df6d0d4044a0ac5e93a6780f864d 2024-11-16T05:39:41,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:39:41,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:41,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:41,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:41,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:41,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/dc69df6d0d4044a0ac5e93a6780f864d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T05:39:42,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/cb0e73e78f9142639be4dfe235692598 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/cb0e73e78f9142639be4dfe235692598 2024-11-16T05:39:42,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/cb0e73e78f9142639be4dfe235692598, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T05:39:42,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3992ee2db1edacc4ba76f5d2411f31eb in 514ms, sequenceid=14, compaction requested=false 2024-11-16T05:39:42,040 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T05:39:42,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:42,061 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:42,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T05:39:42,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/5fa1c4e2801a4ab8922c6096da40ecd6 is 50, key is test_row_0/A:col10/1731735582057/Put/seqid=0 2024-11-16T05:39:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:39:42,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735642094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735642098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735642106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741840_1016 (size=12001) 2024-11-16T05:39:42,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/5fa1c4e2801a4ab8922c6096da40ecd6 2024-11-16T05:39:42,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735642109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735642114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:39:42,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:42,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/3c9c0c32254447f28efd513ba624827c is 50, key is test_row_0/B:col10/1731735582057/Put/seqid=0 2024-11-16T05:39:42,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741841_1017 (size=12001) 2024-11-16T05:39:42,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/3c9c0c32254447f28efd513ba624827c 2024-11-16T05:39:42,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/3701284dfa2047bcacf46b258224328f is 50, key is test_row_0/C:col10/1731735582057/Put/seqid=0 2024-11-16T05:39:42,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735642221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735642222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735642222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741842_1018 (size=12001) 2024-11-16T05:39:42,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/3701284dfa2047bcacf46b258224328f 2024-11-16T05:39:42,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735642240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735642250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/5fa1c4e2801a4ab8922c6096da40ecd6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5fa1c4e2801a4ab8922c6096da40ecd6 2024-11-16T05:39:42,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5fa1c4e2801a4ab8922c6096da40ecd6, entries=150, sequenceid=38, filesize=11.7 K 2024-11-16T05:39:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/3c9c0c32254447f28efd513ba624827c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3c9c0c32254447f28efd513ba624827c 2024-11-16T05:39:42,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3c9c0c32254447f28efd513ba624827c, entries=150, sequenceid=38, filesize=11.7 K 2024-11-16T05:39:42,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/3701284dfa2047bcacf46b258224328f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/3701284dfa2047bcacf46b258224328f 2024-11-16T05:39:42,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:39:42,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:42,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/3701284dfa2047bcacf46b258224328f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-16T05:39:42,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3992ee2db1edacc4ba76f5d2411f31eb in 262ms, sequenceid=38, compaction requested=false 2024-11-16T05:39:42,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:42,446 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:42,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T05:39:42,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/9e5fd900c240470aa946c514af6a5bea is 50, key is test_row_0/A:col10/1731735582446/Put/seqid=0 2024-11-16T05:39:42,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:39:42,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:42,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741843_1019 (size=14341) 2024-11-16T05:39:42,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735642509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735642515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735642511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735642520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735642523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:39:42,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:39:42,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:42,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735642630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735642631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735642631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735642631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735642640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:39:42,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:42,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735642849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735642850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735642850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735642850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735642859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:42,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/9e5fd900c240470aa946c514af6a5bea 2024-11-16T05:39:42,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/71b079b4fdcc4277ae2058bef03a3cf7 is 50, key is test_row_0/B:col10/1731735582446/Put/seqid=0 2024-11-16T05:39:42,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741844_1020 (size=12001) 2024-11-16T05:39:42,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/71b079b4fdcc4277ae2058bef03a3cf7 2024-11-16T05:39:42,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:39:42,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:42,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:42,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/f57b16f04269471cad536c8b6cca2867 is 50, key is test_row_0/C:col10/1731735582446/Put/seqid=0 2024-11-16T05:39:42,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741845_1021 (size=12001) 2024-11-16T05:39:42,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/f57b16f04269471cad536c8b6cca2867 2024-11-16T05:39:43,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/9e5fd900c240470aa946c514af6a5bea as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9e5fd900c240470aa946c514af6a5bea 2024-11-16T05:39:43,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9e5fd900c240470aa946c514af6a5bea, entries=200, sequenceid=53, filesize=14.0 K 2024-11-16T05:39:43,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/71b079b4fdcc4277ae2058bef03a3cf7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/71b079b4fdcc4277ae2058bef03a3cf7 2024-11-16T05:39:43,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/71b079b4fdcc4277ae2058bef03a3cf7, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T05:39:43,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/f57b16f04269471cad536c8b6cca2867 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f57b16f04269471cad536c8b6cca2867 2024-11-16T05:39:43,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f57b16f04269471cad536c8b6cca2867, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T05:39:43,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3992ee2db1edacc4ba76f5d2411f31eb in 619ms, sequenceid=53, compaction requested=true 2024-11-16T05:39:43,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:43,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:43,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:43,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:43,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:43,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:43,086 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:43,090 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:43,092 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:43,094 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:43,094 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:43,095 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:43,095 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/dc69df6d0d4044a0ac5e93a6780f864d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3c9c0c32254447f28efd513ba624827c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/71b079b4fdcc4277ae2058bef03a3cf7] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=35.2 K 2024-11-16T05:39:43,095 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:43,095 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:43,096 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ad9914c52f004ee88c414c128a537730, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5fa1c4e2801a4ab8922c6096da40ecd6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9e5fd900c240470aa946c514af6a5bea] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=37.4 K 2024-11-16T05:39:43,097 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9914c52f004ee88c414c128a537730, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731735581503 2024-11-16T05:39:43,097 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting dc69df6d0d4044a0ac5e93a6780f864d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731735581503 2024-11-16T05:39:43,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:39:43,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:43,099 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c9c0c32254447f28efd513ba624827c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731735581663 2024-11-16T05:39:43,099 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T05:39:43,100 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 71b079b4fdcc4277ae2058bef03a3cf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735582096 2024-11-16T05:39:43,109 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fa1c4e2801a4ab8922c6096da40ecd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731735581663 2024-11-16T05:39:43,110 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e5fd900c240470aa946c514af6a5bea, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735582096 2024-11-16T05:39:43,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/7a41c58b6d3247bf8c961dd58f757e38 is 50, key is test_row_0/A:col10/1731735582506/Put/seqid=0 2024-11-16T05:39:43,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741846_1022 (size=12001) 2024-11-16T05:39:43,158 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/7a41c58b6d3247bf8c961dd58f757e38 2024-11-16T05:39:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:43,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:43,171 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#10 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:43,172 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/5579d2c602084aa29dbd5106687ac5dd is 50, key is test_row_0/A:col10/1731735582446/Put/seqid=0 2024-11-16T05:39:43,183 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#11 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:43,184 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/e789e25f26284daebc4a4c7b3e66bbc0 is 50, key is test_row_0/B:col10/1731735582446/Put/seqid=0 2024-11-16T05:39:43,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/87aa1e35ec4b4f60ad69be9c6ca78ae0 is 50, key is test_row_0/B:col10/1731735582506/Put/seqid=0 2024-11-16T05:39:43,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741847_1023 (size=12104) 2024-11-16T05:39:43,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741849_1025 (size=12001) 2024-11-16T05:39:43,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741848_1024 (size=12104) 2024-11-16T05:39:43,256 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/5579d2c602084aa29dbd5106687ac5dd as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5579d2c602084aa29dbd5106687ac5dd 2024-11-16T05:39:43,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735643245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735643252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735643255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,295 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into 5579d2c602084aa29dbd5106687ac5dd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:43,295 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:43,297 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735583067; duration=0sec 2024-11-16T05:39:43,298 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:43,298 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:43,298 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:43,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735643263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735643264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,306 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:43,306 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:43,306 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:43,307 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/cb0e73e78f9142639be4dfe235692598, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/3701284dfa2047bcacf46b258224328f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f57b16f04269471cad536c8b6cca2867] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=35.2 K 2024-11-16T05:39:43,308 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb0e73e78f9142639be4dfe235692598, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731735581503 2024-11-16T05:39:43,309 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3701284dfa2047bcacf46b258224328f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731735581663 2024-11-16T05:39:43,311 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f57b16f04269471cad536c8b6cca2867, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735582096 2024-11-16T05:39:43,361 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#13 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:43,362 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/f5cbbb582850407b913f2dcd33396325 is 50, key is test_row_0/C:col10/1731735582446/Put/seqid=0 2024-11-16T05:39:43,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735643369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735643379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735643380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741850_1026 (size=12104) 2024-11-16T05:39:43,416 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/f5cbbb582850407b913f2dcd33396325 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f5cbbb582850407b913f2dcd33396325 2024-11-16T05:39:43,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735643409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735643409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,438 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into f5cbbb582850407b913f2dcd33396325(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:43,438 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:43,438 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735583084; duration=0sec 2024-11-16T05:39:43,438 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:43,438 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735643589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735643598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735643599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:39:43,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735643629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735643630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,649 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/87aa1e35ec4b4f60ad69be9c6ca78ae0 2024-11-16T05:39:43,674 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/e789e25f26284daebc4a4c7b3e66bbc0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/e789e25f26284daebc4a4c7b3e66bbc0 2024-11-16T05:39:43,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/5917cf418ec049dc84f57bb0ceab0b04 is 50, key is test_row_0/C:col10/1731735582506/Put/seqid=0 2024-11-16T05:39:43,698 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into e789e25f26284daebc4a4c7b3e66bbc0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:43,699 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:43,699 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735583083; duration=0sec 2024-11-16T05:39:43,699 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:43,699 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:43,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741851_1027 (size=12001) 2024-11-16T05:39:43,756 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/5917cf418ec049dc84f57bb0ceab0b04 2024-11-16T05:39:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/7a41c58b6d3247bf8c961dd58f757e38 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/7a41c58b6d3247bf8c961dd58f757e38 2024-11-16T05:39:43,786 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/7a41c58b6d3247bf8c961dd58f757e38, entries=150, sequenceid=74, filesize=11.7 K 2024-11-16T05:39:43,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/87aa1e35ec4b4f60ad69be9c6ca78ae0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/87aa1e35ec4b4f60ad69be9c6ca78ae0 2024-11-16T05:39:43,816 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/87aa1e35ec4b4f60ad69be9c6ca78ae0, entries=150, sequenceid=74, filesize=11.7 K 2024-11-16T05:39:43,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/5917cf418ec049dc84f57bb0ceab0b04 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5917cf418ec049dc84f57bb0ceab0b04 2024-11-16T05:39:43,832 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5917cf418ec049dc84f57bb0ceab0b04, entries=150, sequenceid=74, filesize=11.7 K 2024-11-16T05:39:43,835 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 3992ee2db1edacc4ba76f5d2411f31eb in 736ms, sequenceid=74, compaction requested=false 2024-11-16T05:39:43,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:43,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:43,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-16T05:39:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-16T05:39:43,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T05:39:43,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3750 sec 2024-11-16T05:39:43,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3990 sec 2024-11-16T05:39:43,903 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:43,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T05:39:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:43,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/d4f471ab3b224cdaa5ef7ebba2d1d9d0 is 50, key is test_row_0/A:col10/1731735583900/Put/seqid=0 2024-11-16T05:39:43,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741852_1028 (size=14341) 2024-11-16T05:39:43,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/d4f471ab3b224cdaa5ef7ebba2d1d9d0 2024-11-16T05:39:43,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735643970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735643975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735643979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735643983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:43,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735643982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/d53ae02c4a924070a37390bcb55aade0 is 50, key is test_row_0/B:col10/1731735583900/Put/seqid=0 2024-11-16T05:39:44,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741853_1029 (size=12001) 2024-11-16T05:39:44,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/d53ae02c4a924070a37390bcb55aade0 2024-11-16T05:39:44,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/321c1a2b40fa4437b36f19f5a1194940 is 50, key is test_row_0/C:col10/1731735583900/Put/seqid=0 2024-11-16T05:39:44,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735644088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735644089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735644098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735644098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735644098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741854_1030 (size=12001) 2024-11-16T05:39:44,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/321c1a2b40fa4437b36f19f5a1194940 2024-11-16T05:39:44,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/d4f471ab3b224cdaa5ef7ebba2d1d9d0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/d4f471ab3b224cdaa5ef7ebba2d1d9d0 2024-11-16T05:39:44,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/d4f471ab3b224cdaa5ef7ebba2d1d9d0, entries=200, sequenceid=94, filesize=14.0 K 2024-11-16T05:39:44,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/d53ae02c4a924070a37390bcb55aade0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/d53ae02c4a924070a37390bcb55aade0 2024-11-16T05:39:44,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/d53ae02c4a924070a37390bcb55aade0, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T05:39:44,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/321c1a2b40fa4437b36f19f5a1194940 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/321c1a2b40fa4437b36f19f5a1194940 2024-11-16T05:39:44,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/321c1a2b40fa4437b36f19f5a1194940, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T05:39:44,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 3992ee2db1edacc4ba76f5d2411f31eb in 292ms, sequenceid=94, compaction requested=true 2024-11-16T05:39:44,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:44,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:44,196 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:44,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:44,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:44,196 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:44,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:44,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:44,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:44,198 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:44,199 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:44,199 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:44,199 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5579d2c602084aa29dbd5106687ac5dd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/7a41c58b6d3247bf8c961dd58f757e38, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/d4f471ab3b224cdaa5ef7ebba2d1d9d0] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=37.5 K 2024-11-16T05:39:44,199 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:44,199 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:44,200 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:44,200 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/e789e25f26284daebc4a4c7b3e66bbc0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/87aa1e35ec4b4f60ad69be9c6ca78ae0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/d53ae02c4a924070a37390bcb55aade0] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=35.3 K 2024-11-16T05:39:44,200 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5579d2c602084aa29dbd5106687ac5dd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735582096 2024-11-16T05:39:44,201 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e789e25f26284daebc4a4c7b3e66bbc0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735582096 2024-11-16T05:39:44,201 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a41c58b6d3247bf8c961dd58f757e38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731735582506 2024-11-16T05:39:44,202 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 87aa1e35ec4b4f60ad69be9c6ca78ae0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731735582506 2024-11-16T05:39:44,202 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4f471ab3b224cdaa5ef7ebba2d1d9d0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731735583239 2024-11-16T05:39:44,203 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting d53ae02c4a924070a37390bcb55aade0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731735583239 2024-11-16T05:39:44,227 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:44,230 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/11cc37003f4c4a1fbeacf6ac1de047be is 50, key is test_row_0/A:col10/1731735583900/Put/seqid=0 2024-11-16T05:39:44,231 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:44,232 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/02e070c21ec84261844281b581461619 is 50, key is test_row_0/B:col10/1731735583900/Put/seqid=0 2024-11-16T05:39:44,267 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:39:44,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741855_1031 (size=12207) 2024-11-16T05:39:44,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741856_1032 (size=12207) 2024-11-16T05:39:44,282 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/11cc37003f4c4a1fbeacf6ac1de047be as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/11cc37003f4c4a1fbeacf6ac1de047be 2024-11-16T05:39:44,299 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into 11cc37003f4c4a1fbeacf6ac1de047be(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:44,300 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:44,300 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735584196; duration=0sec 2024-11-16T05:39:44,300 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:44,300 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:44,300 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:44,302 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:44,302 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:44,302 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:44,303 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f5cbbb582850407b913f2dcd33396325, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5917cf418ec049dc84f57bb0ceab0b04, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/321c1a2b40fa4437b36f19f5a1194940] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=35.3 K 2024-11-16T05:39:44,304 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5cbbb582850407b913f2dcd33396325, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735582096 2024-11-16T05:39:44,305 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5917cf418ec049dc84f57bb0ceab0b04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731735582506 2024-11-16T05:39:44,309 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 321c1a2b40fa4437b36f19f5a1194940, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731735583239 2024-11-16T05:39:44,319 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:44,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T05:39:44,348 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:44,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/f0cf24ab02fc4c0288ee2b811440bdc8 is 50, key is test_row_0/A:col10/1731735584308/Put/seqid=0 2024-11-16T05:39:44,349 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/7e0f338d82004a51951d221f3a1a9b7c is 50, key is test_row_0/C:col10/1731735583900/Put/seqid=0 2024-11-16T05:39:44,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741858_1034 (size=12207) 2024-11-16T05:39:44,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735644349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,370 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T05:39:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741857_1033 (size=12001) 2024-11-16T05:39:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735644363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735644370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735644374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/f0cf24ab02fc4c0288ee2b811440bdc8 2024-11-16T05:39:44,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735644376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,384 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/7e0f338d82004a51951d221f3a1a9b7c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7e0f338d82004a51951d221f3a1a9b7c 2024-11-16T05:39:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,419 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into 7e0f338d82004a51951d221f3a1a9b7c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:44,419 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:44,419 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735584196; duration=0sec 2024-11-16T05:39:44,420 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:44,420 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:44,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/9d03ff69455c433d9816f22cd1194de2 is 50, key is test_row_0/B:col10/1731735584308/Put/seqid=0 2024-11-16T05:39:44,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741859_1035 (size=12001) 2024-11-16T05:39:44,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735644478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735644488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735644488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735644489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735644499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,681 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/02e070c21ec84261844281b581461619 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/02e070c21ec84261844281b581461619 2024-11-16T05:39:44,694 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into 02e070c21ec84261844281b581461619(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:44,694 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:44,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,694 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735584196; duration=0sec 2024-11-16T05:39:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735644689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,694 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:44,694 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735644699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735644698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735644699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735644709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:44,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:44,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:44,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/9d03ff69455c433d9816f22cd1194de2 2024-11-16T05:39:44,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/c11378ad0279453287407de74325fb9a is 50, key is test_row_0/C:col10/1731735584308/Put/seqid=0 2024-11-16T05:39:44,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741860_1036 (size=12001) 2024-11-16T05:39:44,949 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T05:39:44,950 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T05:39:44,952 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:39:44,952 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T05:39:44,952 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T05:39:44,952 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T05:39:44,954 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:39:44,954 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T05:39:45,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735645008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735645019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735645019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735645019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735645019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/c11378ad0279453287407de74325fb9a 2024-11-16T05:39:45,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/f0cf24ab02fc4c0288ee2b811440bdc8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f0cf24ab02fc4c0288ee2b811440bdc8 2024-11-16T05:39:45,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f0cf24ab02fc4c0288ee2b811440bdc8, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T05:39:45,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/9d03ff69455c433d9816f22cd1194de2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/9d03ff69455c433d9816f22cd1194de2 2024-11-16T05:39:45,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/9d03ff69455c433d9816f22cd1194de2, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T05:39:45,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/c11378ad0279453287407de74325fb9a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c11378ad0279453287407de74325fb9a 2024-11-16T05:39:45,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c11378ad0279453287407de74325fb9a, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T05:39:45,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 3992ee2db1edacc4ba76f5d2411f31eb in 1048ms, sequenceid=117, compaction requested=false 2024-11-16T05:39:45,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:45,521 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:45,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T05:39:45,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/5ae92d0ab2d54a108e5bbf877bc3b636 is 50, key is test_row_0/A:col10/1731735585518/Put/seqid=0 2024-11-16T05:39:45,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741861_1037 (size=12147) 2024-11-16T05:39:45,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/5ae92d0ab2d54a108e5bbf877bc3b636 2024-11-16T05:39:45,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/6c6eb72272c7431da062381bd37f6b37 is 50, key is test_row_0/B:col10/1731735585518/Put/seqid=0 2024-11-16T05:39:45,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741862_1038 (size=9757) 2024-11-16T05:39:45,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735645573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735645576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735645579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735645582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735645574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:39:45,629 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:39:45,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:39:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-16T05:39:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:39:45,634 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:39:45,636 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:39:45,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:39:45,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735645698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735645699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735645699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735645699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735645699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:39:45,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:39:45,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:45,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:45,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:45,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:45,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735645909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735645910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735645910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735645915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735645915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:45,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:39:45,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:45,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:45,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:45,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:45,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:39:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:45,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/6c6eb72272c7431da062381bd37f6b37 2024-11-16T05:39:46,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/94347bdd1ea44220b698229c0359c192 is 50, key is test_row_0/C:col10/1731735585518/Put/seqid=0 2024-11-16T05:39:46,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741863_1039 (size=9757) 2024-11-16T05:39:46,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/94347bdd1ea44220b698229c0359c192 2024-11-16T05:39:46,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/5ae92d0ab2d54a108e5bbf877bc3b636 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5ae92d0ab2d54a108e5bbf877bc3b636 2024-11-16T05:39:46,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5ae92d0ab2d54a108e5bbf877bc3b636, entries=150, sequenceid=135, filesize=11.9 K 2024-11-16T05:39:46,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/6c6eb72272c7431da062381bd37f6b37 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6c6eb72272c7431da062381bd37f6b37 2024-11-16T05:39:46,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6c6eb72272c7431da062381bd37f6b37, entries=100, sequenceid=135, filesize=9.5 K 2024-11-16T05:39:46,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/94347bdd1ea44220b698229c0359c192 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/94347bdd1ea44220b698229c0359c192 2024-11-16T05:39:46,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/94347bdd1ea44220b698229c0359c192, entries=100, sequenceid=135, filesize=9.5 K 2024-11-16T05:39:46,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3992ee2db1edacc4ba76f5d2411f31eb in 567ms, sequenceid=135, compaction requested=true 2024-11-16T05:39:46,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:46,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:46,088 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:46,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:46,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:46,088 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:46,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:46,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:46,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:46,091 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:46,091 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:46,092 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:46,092 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:46,092 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:46,092 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:46,092 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/11cc37003f4c4a1fbeacf6ac1de047be, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f0cf24ab02fc4c0288ee2b811440bdc8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5ae92d0ab2d54a108e5bbf877bc3b636] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=35.5 K 2024-11-16T05:39:46,092 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/02e070c21ec84261844281b581461619, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/9d03ff69455c433d9816f22cd1194de2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6c6eb72272c7431da062381bd37f6b37] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=33.2 K 2024-11-16T05:39:46,093 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 02e070c21ec84261844281b581461619, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731735583239 2024-11-16T05:39:46,093 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11cc37003f4c4a1fbeacf6ac1de047be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731735583239 2024-11-16T05:39:46,094 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d03ff69455c433d9816f22cd1194de2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731735583949 2024-11-16T05:39:46,094 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0cf24ab02fc4c0288ee2b811440bdc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731735583949 2024-11-16T05:39:46,095 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c6eb72272c7431da062381bd37f6b37, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731735584368 2024-11-16T05:39:46,096 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae92d0ab2d54a108e5bbf877bc3b636, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731735584366 2024-11-16T05:39:46,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:39:46,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:46,105 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T05:39:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e34a3a50f2e145018ea80179d9bbc615 is 50, key is test_row_0/A:col10/1731735585575/Put/seqid=0 2024-11-16T05:39:46,130 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:46,131 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/1785f360c27a4291936f11902a8b0ae9 is 50, key is test_row_0/B:col10/1731735585518/Put/seqid=0 2024-11-16T05:39:46,139 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#29 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:46,140 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/96974b5534c541a294107894c23efbc2 is 50, key is test_row_0/A:col10/1731735585518/Put/seqid=0 2024-11-16T05:39:46,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741865_1041 (size=12409) 2024-11-16T05:39:46,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741864_1040 (size=12151) 2024-11-16T05:39:46,199 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e34a3a50f2e145018ea80179d9bbc615 2024-11-16T05:39:46,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741866_1042 (size=12409) 2024-11-16T05:39:46,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/18a4046c07214170a35dfee847c58a68 is 50, key is test_row_0/B:col10/1731735585575/Put/seqid=0 2024-11-16T05:39:46,216 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/96974b5534c541a294107894c23efbc2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/96974b5534c541a294107894c23efbc2 2024-11-16T05:39:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:46,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:46,233 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into 96974b5534c541a294107894c23efbc2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:46,233 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:46,233 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735586088; duration=0sec 2024-11-16T05:39:46,233 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:46,233 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:46,233 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:46,235 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:46,235 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:46,235 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:46,236 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7e0f338d82004a51951d221f3a1a9b7c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c11378ad0279453287407de74325fb9a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/94347bdd1ea44220b698229c0359c192] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=33.2 K 2024-11-16T05:39:46,236 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e0f338d82004a51951d221f3a1a9b7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731735583239 2024-11-16T05:39:46,237 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c11378ad0279453287407de74325fb9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731735583949 2024-11-16T05:39:46,239 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94347bdd1ea44220b698229c0359c192, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731735584368 2024-11-16T05:39:46,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741867_1043 (size=12151) 2024-11-16T05:39:46,244 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/18a4046c07214170a35dfee847c58a68 2024-11-16T05:39:46,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/ff4deb02f1b9422a8957dcbd4bd64f1f is 50, key is test_row_0/C:col10/1731735585575/Put/seqid=0 2024-11-16T05:39:46,258 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:39:46,259 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/e732865aa73343eda8e66a942d3d73a0 is 50, key is test_row_0/C:col10/1731735585518/Put/seqid=0 2024-11-16T05:39:46,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735646249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735646252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735646258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735646265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735646267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741868_1044 (size=12151) 2024-11-16T05:39:46,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741869_1045 (size=12409) 2024-11-16T05:39:46,302 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/e732865aa73343eda8e66a942d3d73a0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/e732865aa73343eda8e66a942d3d73a0 2024-11-16T05:39:46,313 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into e732865aa73343eda8e66a942d3d73a0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:46,313 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:46,313 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735586088; duration=0sec 2024-11-16T05:39:46,313 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:46,313 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:46,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735646368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735646378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735646378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735646379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735646382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735646579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735646588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735646588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735646588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,603 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/1785f360c27a4291936f11902a8b0ae9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/1785f360c27a4291936f11902a8b0ae9 2024-11-16T05:39:46,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735646599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,615 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into 1785f360c27a4291936f11902a8b0ae9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:46,615 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:46,615 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735586088; duration=0sec 2024-11-16T05:39:46,615 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:46,615 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:46,686 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/ff4deb02f1b9422a8957dcbd4bd64f1f 2024-11-16T05:39:46,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e34a3a50f2e145018ea80179d9bbc615 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e34a3a50f2e145018ea80179d9bbc615 2024-11-16T05:39:46,706 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e34a3a50f2e145018ea80179d9bbc615, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T05:39:46,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/18a4046c07214170a35dfee847c58a68 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/18a4046c07214170a35dfee847c58a68 2024-11-16T05:39:46,717 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/18a4046c07214170a35dfee847c58a68, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T05:39:46,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/ff4deb02f1b9422a8957dcbd4bd64f1f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ff4deb02f1b9422a8957dcbd4bd64f1f 2024-11-16T05:39:46,727 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ff4deb02f1b9422a8957dcbd4bd64f1f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T05:39:46,729 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 3992ee2db1edacc4ba76f5d2411f31eb in 623ms, sequenceid=155, compaction requested=false 2024-11-16T05:39:46,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:46,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:46,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-16T05:39:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-16T05:39:46,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-16T05:39:46,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0960 sec 2024-11-16T05:39:46,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1100 sec 2024-11-16T05:39:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:39:46,769 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:39:46,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:39:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-16T05:39:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:39:46,772 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:39:46,773 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:39:46,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:39:46,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:39:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:46,891 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:46,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T05:39:46,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/9bc048fba9ae4c5eac2b89db6e75b0d6 is 50, key is test_row_0/A:col10/1731735586238/Put/seqid=0 2024-11-16T05:39:46,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:39:46,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:46,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:46,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735646924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741870_1046 (size=12151) 2024-11-16T05:39:46,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/9bc048fba9ae4c5eac2b89db6e75b0d6 2024-11-16T05:39:46,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735646926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735646928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735646931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735646932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:46,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:46,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:46,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/ad56ed6f566947d4ae1eaafbe76818e4 is 50, key is test_row_0/B:col10/1731735586238/Put/seqid=0 2024-11-16T05:39:46,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741871_1047 (size=12151) 2024-11-16T05:39:46,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/ad56ed6f566947d4ae1eaafbe76818e4 2024-11-16T05:39:46,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/7c5aa4ffa00b47d786d2e21c587051fa is 50, key is test_row_0/C:col10/1731735586238/Put/seqid=0 2024-11-16T05:39:47,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741872_1048 (size=12151) 2024-11-16T05:39:47,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/7c5aa4ffa00b47d786d2e21c587051fa 2024-11-16T05:39:47,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/9bc048fba9ae4c5eac2b89db6e75b0d6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9bc048fba9ae4c5eac2b89db6e75b0d6 2024-11-16T05:39:47,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735647039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9bc048fba9ae4c5eac2b89db6e75b0d6, entries=150, sequenceid=176, filesize=11.9 K 2024-11-16T05:39:47,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/ad56ed6f566947d4ae1eaafbe76818e4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ad56ed6f566947d4ae1eaafbe76818e4 2024-11-16T05:39:47,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735647049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735647049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735647049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735647049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ad56ed6f566947d4ae1eaafbe76818e4, entries=150, sequenceid=176, filesize=11.9 K 2024-11-16T05:39:47,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/7c5aa4ffa00b47d786d2e21c587051fa as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7c5aa4ffa00b47d786d2e21c587051fa 2024-11-16T05:39:47,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7c5aa4ffa00b47d786d2e21c587051fa, entries=150, sequenceid=176, filesize=11.9 K 2024-11-16T05:39:47,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 3992ee2db1edacc4ba76f5d2411f31eb in 184ms, sequenceid=176, compaction requested=true 2024-11-16T05:39:47,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:47,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:47,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:47,075 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:47,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:47,075 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:47,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:47,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:47,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:47,077 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:47,077 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:47,077 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:47,077 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/1785f360c27a4291936f11902a8b0ae9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/18a4046c07214170a35dfee847c58a68, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ad56ed6f566947d4ae1eaafbe76818e4] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=35.9 K 2024-11-16T05:39:47,078 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1785f360c27a4291936f11902a8b0ae9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731735584312 2024-11-16T05:39:47,080 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a4046c07214170a35dfee847c58a68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731735585562 2024-11-16T05:39:47,080 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:47,080 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:47,080 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:47,080 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/96974b5534c541a294107894c23efbc2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e34a3a50f2e145018ea80179d9bbc615, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9bc048fba9ae4c5eac2b89db6e75b0d6] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=35.9 K 2024-11-16T05:39:47,081 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ad56ed6f566947d4ae1eaafbe76818e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731735586238 2024-11-16T05:39:47,081 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96974b5534c541a294107894c23efbc2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731735584312 2024-11-16T05:39:47,081 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e34a3a50f2e145018ea80179d9bbc615, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731735585562 2024-11-16T05:39:47,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:39:47,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:47,083 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T05:39:47,085 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bc048fba9ae4c5eac2b89db6e75b0d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731735586238 2024-11-16T05:39:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:39:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/a8cd760548b54b0c959660bcd884bb5f is 50, key is test_row_0/A:col10/1731735586925/Put/seqid=0 2024-11-16T05:39:47,128 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:47,128 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/f421913d38344673bc6af2b180cb2ed6 is 50, key is test_row_0/A:col10/1731735586238/Put/seqid=0 2024-11-16T05:39:47,135 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:47,136 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/e3cdcc5a3cf0484db93e3aa238e8fe39 is 50, key is test_row_0/B:col10/1731735586238/Put/seqid=0 2024-11-16T05:39:47,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741873_1049 (size=12151) 2024-11-16T05:39:47,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741874_1050 (size=12561) 2024-11-16T05:39:47,168 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/f421913d38344673bc6af2b180cb2ed6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f421913d38344673bc6af2b180cb2ed6 2024-11-16T05:39:47,178 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into f421913d38344673bc6af2b180cb2ed6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:47,178 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:47,178 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735587075; duration=0sec 2024-11-16T05:39:47,178 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:47,178 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:47,178 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:47,181 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:47,181 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:47,181 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:47,182 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/e732865aa73343eda8e66a942d3d73a0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ff4deb02f1b9422a8957dcbd4bd64f1f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7c5aa4ffa00b47d786d2e21c587051fa] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=35.9 K 2024-11-16T05:39:47,182 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e732865aa73343eda8e66a942d3d73a0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731735584312 2024-11-16T05:39:47,183 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff4deb02f1b9422a8957dcbd4bd64f1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731735585562 2024-11-16T05:39:47,184 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c5aa4ffa00b47d786d2e21c587051fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731735586238 2024-11-16T05:39:47,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741875_1051 (size=12561) 2024-11-16T05:39:47,215 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:47,216 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/e66f487470f548a183601211e135d780 is 50, key is test_row_0/C:col10/1731735586238/Put/seqid=0 2024-11-16T05:39:47,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741876_1052 (size=12561) 2024-11-16T05:39:47,249 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/e66f487470f548a183601211e135d780 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/e66f487470f548a183601211e135d780 2024-11-16T05:39:47,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:47,267 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into e66f487470f548a183601211e135d780(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:47,271 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:47,271 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735587076; duration=0sec 2024-11-16T05:39:47,271 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:47,271 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:47,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735647280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735647283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735647285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735647287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735647289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:39:47,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735647399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735647399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735647399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735647399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735647399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,542 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/a8cd760548b54b0c959660bcd884bb5f 2024-11-16T05:39:47,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/aa219ec622944c598b152f056e30a07d is 50, key is test_row_0/B:col10/1731735586925/Put/seqid=0 2024-11-16T05:39:47,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741877_1053 (size=12151) 2024-11-16T05:39:47,596 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/e3cdcc5a3cf0484db93e3aa238e8fe39 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/e3cdcc5a3cf0484db93e3aa238e8fe39 2024-11-16T05:39:47,605 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into e3cdcc5a3cf0484db93e3aa238e8fe39(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:47,605 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:47,605 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735587075; duration=0sec 2024-11-16T05:39:47,605 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:47,605 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735647608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735647608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735647608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735647619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735647619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:39:47,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735647918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735647918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735647918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735647928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735647928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:47,963 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/aa219ec622944c598b152f056e30a07d 2024-11-16T05:39:47,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/a34a4730577149af8bf7ca8bebb56806 is 50, key is test_row_0/C:col10/1731735586925/Put/seqid=0 2024-11-16T05:39:48,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741878_1054 (size=12151) 2024-11-16T05:39:48,009 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/a34a4730577149af8bf7ca8bebb56806 2024-11-16T05:39:48,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/a8cd760548b54b0c959660bcd884bb5f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a8cd760548b54b0c959660bcd884bb5f 2024-11-16T05:39:48,026 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a8cd760548b54b0c959660bcd884bb5f, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T05:39:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/aa219ec622944c598b152f056e30a07d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/aa219ec622944c598b152f056e30a07d 2024-11-16T05:39:48,048 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/aa219ec622944c598b152f056e30a07d, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T05:39:48,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/a34a4730577149af8bf7ca8bebb56806 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/a34a4730577149af8bf7ca8bebb56806 2024-11-16T05:39:48,063 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/a34a4730577149af8bf7ca8bebb56806, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T05:39:48,066 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 3992ee2db1edacc4ba76f5d2411f31eb in 982ms, sequenceid=194, compaction requested=false 2024-11-16T05:39:48,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:48,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:48,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-16T05:39:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-16T05:39:48,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-16T05:39:48,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2960 sec 2024-11-16T05:39:48,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3070 sec 2024-11-16T05:39:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:48,432 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:48,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T05:39:48,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/f292166f9b184bceb9cda79c0596a89c is 50, key is test_row_0/A:col10/1731735588432/Put/seqid=0 2024-11-16T05:39:48,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735648451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735648458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735648459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735648461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735648461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741879_1055 (size=12151) 2024-11-16T05:39:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735648570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735648571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735648571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735648580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735648580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735648779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735648789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735648788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735648789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735648789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:48,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/f292166f9b184bceb9cda79c0596a89c 2024-11-16T05:39:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:39:48,919 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:39:48,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/38ba4342703f44e19fd9ce5d3b75b75c is 50, key is test_row_0/B:col10/1731735588432/Put/seqid=0 2024-11-16T05:39:48,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:39:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-16T05:39:48,924 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:39:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:39:48,927 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:39:48,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:39:48,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741880_1056 (size=12151) 2024-11-16T05:39:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:39:49,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:39:49,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:49,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:49,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:49,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:49,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:49,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735649098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735649098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735649098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735649108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735649108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:39:49,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:39:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:49,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:49,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:49,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/38ba4342703f44e19fd9ce5d3b75b75c 2024-11-16T05:39:49,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/978b7d4b012d45f4a7eec1e488b571c6 is 50, key is test_row_0/C:col10/1731735588432/Put/seqid=0 2024-11-16T05:39:49,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741881_1057 (size=12151) 2024-11-16T05:39:49,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:39:49,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:49,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:49,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:49,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:49,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:49,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/978b7d4b012d45f4a7eec1e488b571c6 2024-11-16T05:39:49,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/f292166f9b184bceb9cda79c0596a89c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f292166f9b184bceb9cda79c0596a89c 2024-11-16T05:39:49,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f292166f9b184bceb9cda79c0596a89c, entries=150, sequenceid=218, filesize=11.9 K 2024-11-16T05:39:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/38ba4342703f44e19fd9ce5d3b75b75c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/38ba4342703f44e19fd9ce5d3b75b75c 2024-11-16T05:39:49,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/38ba4342703f44e19fd9ce5d3b75b75c, entries=150, sequenceid=218, filesize=11.9 K 2024-11-16T05:39:49,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/978b7d4b012d45f4a7eec1e488b571c6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/978b7d4b012d45f4a7eec1e488b571c6 2024-11-16T05:39:49,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/978b7d4b012d45f4a7eec1e488b571c6, entries=150, sequenceid=218, filesize=11.9 K 2024-11-16T05:39:49,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3992ee2db1edacc4ba76f5d2411f31eb in 1030ms, sequenceid=218, compaction requested=true 2024-11-16T05:39:49,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:49,463 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:49,463 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:49,464 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:49,464 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:49,464 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:49,465 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f421913d38344673bc6af2b180cb2ed6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a8cd760548b54b0c959660bcd884bb5f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f292166f9b184bceb9cda79c0596a89c] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.0 K 2024-11-16T05:39:49,465 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:49,465 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:49,465 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:49,465 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/e3cdcc5a3cf0484db93e3aa238e8fe39, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/aa219ec622944c598b152f056e30a07d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/38ba4342703f44e19fd9ce5d3b75b75c] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.0 K 2024-11-16T05:39:49,470 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f421913d38344673bc6af2b180cb2ed6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731735586238 2024-11-16T05:39:49,470 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e3cdcc5a3cf0484db93e3aa238e8fe39, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731735586238 2024-11-16T05:39:49,471 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting aa219ec622944c598b152f056e30a07d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731735586925 2024-11-16T05:39:49,471 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8cd760548b54b0c959660bcd884bb5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731735586925 2024-11-16T05:39:49,471 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ba4342703f44e19fd9ce5d3b75b75c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731735588431 2024-11-16T05:39:49,471 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f292166f9b184bceb9cda79c0596a89c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731735588431 2024-11-16T05:39:49,484 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:49,485 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:49,485 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b7d2b92a3f964a01afcf039ca33dd114 is 50, key is test_row_0/A:col10/1731735588432/Put/seqid=0 2024-11-16T05:39:49,486 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/ef3168b17d2c44a3924cbee21f2747b5 is 50, key is test_row_0/B:col10/1731735588432/Put/seqid=0 2024-11-16T05:39:49,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741882_1058 (size=12663) 2024-11-16T05:39:49,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741883_1059 (size=12663) 2024-11-16T05:39:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:39:49,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:39:49,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:49,560 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T05:39:49,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/1513eab2a2c04770a63a52230190a441 is 50, key is test_row_0/A:col10/1731735588458/Put/seqid=0 2024-11-16T05:39:49,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741884_1060 (size=12151) 2024-11-16T05:39:49,573 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/1513eab2a2c04770a63a52230190a441 2024-11-16T05:39:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/2d92f19dba964766a7b453468d6f1a3e is 50, key is test_row_0/B:col10/1731735588458/Put/seqid=0 2024-11-16T05:39:49,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:49,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741885_1061 (size=12151) 2024-11-16T05:39:49,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735649642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735649642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735649647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735649648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735649650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735649758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735649758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735649768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735649768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735649768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,918 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/ef3168b17d2c44a3924cbee21f2747b5 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ef3168b17d2c44a3924cbee21f2747b5 2024-11-16T05:39:49,918 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b7d2b92a3f964a01afcf039ca33dd114 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b7d2b92a3f964a01afcf039ca33dd114 2024-11-16T05:39:49,927 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into ef3168b17d2c44a3924cbee21f2747b5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:49,927 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:49,927 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735589463; duration=0sec 2024-11-16T05:39:49,927 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:49,927 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:49,927 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:49,929 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into b7d2b92a3f964a01afcf039ca33dd114(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:49,929 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:49,929 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735589463; duration=0sec 2024-11-16T05:39:49,929 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:49,929 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:49,930 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:49,931 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:49,931 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:49,931 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/e66f487470f548a183601211e135d780, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/a34a4730577149af8bf7ca8bebb56806, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/978b7d4b012d45f4a7eec1e488b571c6] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.0 K 2024-11-16T05:39:49,931 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e66f487470f548a183601211e135d780, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731735586238 2024-11-16T05:39:49,932 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting a34a4730577149af8bf7ca8bebb56806, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731735586925 2024-11-16T05:39:49,932 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 978b7d4b012d45f4a7eec1e488b571c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731735588431 2024-11-16T05:39:49,954 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:49,955 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/8db3525a7b8943d9a1cd978657f5287e is 50, key is test_row_0/C:col10/1731735588432/Put/seqid=0 2024-11-16T05:39:49,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735649968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735649968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735649979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735649979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735649980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741886_1062 (size=12663) 2024-11-16T05:39:50,013 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/8db3525a7b8943d9a1cd978657f5287e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/8db3525a7b8943d9a1cd978657f5287e 2024-11-16T05:39:50,029 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into 8db3525a7b8943d9a1cd978657f5287e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:50,029 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:50,029 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735589463; duration=0sec 2024-11-16T05:39:50,029 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:50,030 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:50,032 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/2d92f19dba964766a7b453468d6f1a3e 2024-11-16T05:39:50,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/ac11e1f97a744cc8a22820e479fa2f3e is 50, key is test_row_0/C:col10/1731735588458/Put/seqid=0 2024-11-16T05:39:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:39:50,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741887_1063 (size=12151) 2024-11-16T05:39:50,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735650280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735650281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735650305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735650305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735650305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,470 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/ac11e1f97a744cc8a22820e479fa2f3e 2024-11-16T05:39:50,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/1513eab2a2c04770a63a52230190a441 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/1513eab2a2c04770a63a52230190a441 2024-11-16T05:39:50,488 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/1513eab2a2c04770a63a52230190a441, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T05:39:50,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/2d92f19dba964766a7b453468d6f1a3e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2d92f19dba964766a7b453468d6f1a3e 2024-11-16T05:39:50,498 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2d92f19dba964766a7b453468d6f1a3e, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T05:39:50,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/ac11e1f97a744cc8a22820e479fa2f3e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ac11e1f97a744cc8a22820e479fa2f3e 2024-11-16T05:39:50,505 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ac11e1f97a744cc8a22820e479fa2f3e, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T05:39:50,506 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3992ee2db1edacc4ba76f5d2411f31eb in 946ms, sequenceid=233, compaction requested=false 2024-11-16T05:39:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-16T05:39:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-16T05:39:50,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T05:39:50,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5820 sec 2024-11-16T05:39:50,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5930 sec 2024-11-16T05:39:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:50,791 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:50,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T05:39:50,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/892eb8574dbb440ba5b6f964caaec63a is 50, key is test_row_0/A:col10/1731735589648/Put/seqid=0 2024-11-16T05:39:50,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735650828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741888_1064 (size=12201) 2024-11-16T05:39:50,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735650830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735650831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735650831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735650833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735650938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735650948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735650948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735650948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:50,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735650948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:39:51,069 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:39:51,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:39:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-16T05:39:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:39:51,073 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:39:51,075 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:39:51,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:39:51,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735651148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735651158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735651158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735651158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735651159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:39:51,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T05:39:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:51,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:51,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/892eb8574dbb440ba5b6f964caaec63a 2024-11-16T05:39:51,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/4bbb15cfc10b4d2ab5930956fa3ae3c4 is 50, key is test_row_0/B:col10/1731735589648/Put/seqid=0 2024-11-16T05:39:51,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741889_1065 (size=12201) 2024-11-16T05:39:51,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/4bbb15cfc10b4d2ab5930956fa3ae3c4 2024-11-16T05:39:51,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/4e6d7bb7da944ffa9b5a69c37df3fba3 is 50, key is test_row_0/C:col10/1731735589648/Put/seqid=0 2024-11-16T05:39:51,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741890_1066 (size=12201) 2024-11-16T05:39:51,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/4e6d7bb7da944ffa9b5a69c37df3fba3 2024-11-16T05:39:51,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/892eb8574dbb440ba5b6f964caaec63a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/892eb8574dbb440ba5b6f964caaec63a 2024-11-16T05:39:51,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/892eb8574dbb440ba5b6f964caaec63a, entries=150, sequenceid=258, filesize=11.9 K 2024-11-16T05:39:51,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/4bbb15cfc10b4d2ab5930956fa3ae3c4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4bbb15cfc10b4d2ab5930956fa3ae3c4 2024-11-16T05:39:51,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4bbb15cfc10b4d2ab5930956fa3ae3c4, entries=150, sequenceid=258, filesize=11.9 K 2024-11-16T05:39:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/4e6d7bb7da944ffa9b5a69c37df3fba3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4e6d7bb7da944ffa9b5a69c37df3fba3 2024-11-16T05:39:51,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4e6d7bb7da944ffa9b5a69c37df3fba3, entries=150, sequenceid=258, filesize=11.9 K 2024-11-16T05:39:51,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3992ee2db1edacc4ba76f5d2411f31eb in 553ms, sequenceid=258, compaction requested=true 2024-11-16T05:39:51,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:51,345 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:51,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:51,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:51,345 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:51,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:51,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:51,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:51,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:51,346 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:51,347 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:51,347 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:51,347 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b7d2b92a3f964a01afcf039ca33dd114, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/1513eab2a2c04770a63a52230190a441, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/892eb8574dbb440ba5b6f964caaec63a] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.1 K 2024-11-16T05:39:51,347 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:51,347 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:51,347 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:51,348 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ef3168b17d2c44a3924cbee21f2747b5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2d92f19dba964766a7b453468d6f1a3e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4bbb15cfc10b4d2ab5930956fa3ae3c4] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.1 K 2024-11-16T05:39:51,348 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7d2b92a3f964a01afcf039ca33dd114, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731735588431 2024-11-16T05:39:51,348 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ef3168b17d2c44a3924cbee21f2747b5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731735588431 2024-11-16T05:39:51,348 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1513eab2a2c04770a63a52230190a441, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731735588450 2024-11-16T05:39:51,349 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d92f19dba964766a7b453468d6f1a3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731735588450 2024-11-16T05:39:51,349 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 892eb8574dbb440ba5b6f964caaec63a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731735589645 2024-11-16T05:39:51,349 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bbb15cfc10b4d2ab5930956fa3ae3c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731735589645 2024-11-16T05:39:51,366 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:51,367 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#54 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:51,367 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/566962c3a4544487932d7a19a35ef8a3 is 50, key is test_row_0/B:col10/1731735589648/Put/seqid=0 2024-11-16T05:39:51,367 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/a45fe28fc8284d8aa27e37a3193d889c is 50, key is test_row_0/A:col10/1731735589648/Put/seqid=0 2024-11-16T05:39:51,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T05:39:51,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:39:51,388 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T05:39:51,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741892_1068 (size=12815) 2024-11-16T05:39:51,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741891_1067 (size=12815) 2024-11-16T05:39:51,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/4806287c044f49d49aa56d05bfd3492d is 50, key is test_row_0/A:col10/1731735590829/Put/seqid=0 2024-11-16T05:39:51,409 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/566962c3a4544487932d7a19a35ef8a3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/566962c3a4544487932d7a19a35ef8a3 2024-11-16T05:39:51,420 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into 566962c3a4544487932d7a19a35ef8a3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:51,421 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:51,421 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735591345; duration=0sec 2024-11-16T05:39:51,421 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:51,421 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:51,421 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:51,423 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:51,423 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:51,423 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:51,424 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/8db3525a7b8943d9a1cd978657f5287e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ac11e1f97a744cc8a22820e479fa2f3e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4e6d7bb7da944ffa9b5a69c37df3fba3] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.1 K 2024-11-16T05:39:51,424 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db3525a7b8943d9a1cd978657f5287e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731735588431 2024-11-16T05:39:51,424 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ac11e1f97a744cc8a22820e479fa2f3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731735588450 2024-11-16T05:39:51,425 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e6d7bb7da944ffa9b5a69c37df3fba3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731735589645 2024-11-16T05:39:51,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741893_1069 (size=12301) 2024-11-16T05:39:51,428 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/4806287c044f49d49aa56d05bfd3492d 2024-11-16T05:39:51,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/27e8e697b873474aae3342aca7ba77b9 is 50, key is test_row_0/B:col10/1731735590829/Put/seqid=0 2024-11-16T05:39:51,455 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:51,456 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/2096c60c4da04725b704d2e812b16a63 is 50, key is test_row_0/C:col10/1731735589648/Put/seqid=0 2024-11-16T05:39:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:51,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:51,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741894_1070 (size=12301) 2024-11-16T05:39:51,472 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/27e8e697b873474aae3342aca7ba77b9 2024-11-16T05:39:51,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741895_1071 (size=12815) 2024-11-16T05:39:51,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/6b50d90db79f4c6a8189b0febee877c4 is 50, key is test_row_0/C:col10/1731735590829/Put/seqid=0 2024-11-16T05:39:51,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735651495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735651495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735651498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735651498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735651499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741896_1072 (size=12301) 2024-11-16T05:39:51,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735651608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735651608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735651608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735651608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735651608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:39:51,797 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/a45fe28fc8284d8aa27e37a3193d889c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a45fe28fc8284d8aa27e37a3193d889c 2024-11-16T05:39:51,803 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into a45fe28fc8284d8aa27e37a3193d889c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:51,803 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:51,803 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735591345; duration=0sec 2024-11-16T05:39:51,804 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:51,804 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:51,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735651818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735651819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735651819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735651819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735651819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:51,883 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/2096c60c4da04725b704d2e812b16a63 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/2096c60c4da04725b704d2e812b16a63 2024-11-16T05:39:51,890 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into 2096c60c4da04725b704d2e812b16a63(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:51,890 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:51,890 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735591346; duration=0sec 2024-11-16T05:39:51,890 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:51,890 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:51,910 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/6b50d90db79f4c6a8189b0febee877c4 2024-11-16T05:39:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/4806287c044f49d49aa56d05bfd3492d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4806287c044f49d49aa56d05bfd3492d 2024-11-16T05:39:51,940 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4806287c044f49d49aa56d05bfd3492d, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T05:39:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/27e8e697b873474aae3342aca7ba77b9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/27e8e697b873474aae3342aca7ba77b9 2024-11-16T05:39:51,948 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/27e8e697b873474aae3342aca7ba77b9, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T05:39:51,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/6b50d90db79f4c6a8189b0febee877c4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/6b50d90db79f4c6a8189b0febee877c4 2024-11-16T05:39:51,959 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/6b50d90db79f4c6a8189b0febee877c4, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T05:39:51,960 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3992ee2db1edacc4ba76f5d2411f31eb in 572ms, sequenceid=272, compaction requested=false 2024-11-16T05:39:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-16T05:39:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-16T05:39:51,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-16T05:39:51,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 887 msec 2024-11-16T05:39:51,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 897 msec 2024-11-16T05:39:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:52,130 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:52,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T05:39:52,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/9183f6ca411d46f48574b2e72f7157c5 is 50, key is test_row_0/A:col10/1731735591476/Put/seqid=0 2024-11-16T05:39:52,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741897_1073 (size=12301) 2024-11-16T05:39:52,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/9183f6ca411d46f48574b2e72f7157c5 2024-11-16T05:39:52,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735652144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735652144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735652150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735652151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735652151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/027468104b9d47bc868667c53ba0f127 is 50, key is test_row_0/B:col10/1731735591476/Put/seqid=0 2024-11-16T05:39:52,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741898_1074 (size=12301) 2024-11-16T05:39:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:39:52,209 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:39:52,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:39:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-16T05:39:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:39:52,211 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:39:52,212 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:39:52,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:39:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735652258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735652258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735652259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735652258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735652259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:39:52,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T05:39:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:52,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:52,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735652468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735652469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735652469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735652469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735652469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T05:39:52,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:52,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:52,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:52,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:52,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:39:52,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/027468104b9d47bc868667c53ba0f127 2024-11-16T05:39:52,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/81912b1b9e6d46c08654c0b183a92001 is 50, key is test_row_0/C:col10/1731735591476/Put/seqid=0 2024-11-16T05:39:52,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741899_1075 (size=12301) 2024-11-16T05:39:52,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/81912b1b9e6d46c08654c0b183a92001 2024-11-16T05:39:52,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/9183f6ca411d46f48574b2e72f7157c5 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9183f6ca411d46f48574b2e72f7157c5 2024-11-16T05:39:52,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9183f6ca411d46f48574b2e72f7157c5, entries=150, sequenceid=298, filesize=12.0 K 2024-11-16T05:39:52,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/027468104b9d47bc868667c53ba0f127 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/027468104b9d47bc868667c53ba0f127 2024-11-16T05:39:52,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/027468104b9d47bc868667c53ba0f127, entries=150, sequenceid=298, filesize=12.0 K 2024-11-16T05:39:52,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/81912b1b9e6d46c08654c0b183a92001 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81912b1b9e6d46c08654c0b183a92001 2024-11-16T05:39:52,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81912b1b9e6d46c08654c0b183a92001, entries=150, sequenceid=298, filesize=12.0 K 2024-11-16T05:39:52,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3992ee2db1edacc4ba76f5d2411f31eb in 489ms, sequenceid=298, compaction requested=true 2024-11-16T05:39:52,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:52,619 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:52,620 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:52,621 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:52,621 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:52,621 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:52,621 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a45fe28fc8284d8aa27e37a3193d889c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4806287c044f49d49aa56d05bfd3492d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9183f6ca411d46f48574b2e72f7157c5] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.5 K 2024-11-16T05:39:52,622 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:52,622 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:52,622 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:52,622 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/566962c3a4544487932d7a19a35ef8a3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/27e8e697b873474aae3342aca7ba77b9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/027468104b9d47bc868667c53ba0f127] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.5 K 2024-11-16T05:39:52,622 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a45fe28fc8284d8aa27e37a3193d889c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731735589645 2024-11-16T05:39:52,622 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4806287c044f49d49aa56d05bfd3492d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731735590821 2024-11-16T05:39:52,622 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 566962c3a4544487932d7a19a35ef8a3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731735589645 2024-11-16T05:39:52,623 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9183f6ca411d46f48574b2e72f7157c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731735591476 2024-11-16T05:39:52,623 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 27e8e697b873474aae3342aca7ba77b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731735590821 2024-11-16T05:39:52,623 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 027468104b9d47bc868667c53ba0f127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731735591476 2024-11-16T05:39:52,632 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:52,633 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/6b7d6c09334a42d4a53ca441c055e4b4 is 50, key is test_row_0/A:col10/1731735591476/Put/seqid=0 2024-11-16T05:39:52,635 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:52,636 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/449dd95578154edda67c11a449fea073 is 50, key is test_row_0/B:col10/1731735591476/Put/seqid=0 2024-11-16T05:39:52,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741900_1076 (size=13017) 2024-11-16T05:39:52,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T05:39:52,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:52,672 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T05:39:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/722a56f90c154cf895c32f119c0a1f1b is 50, key is test_row_0/A:col10/1731735592135/Put/seqid=0 2024-11-16T05:39:52,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741901_1077 (size=13017) 2024-11-16T05:39:52,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741902_1078 (size=12301) 2024-11-16T05:39:52,688 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/722a56f90c154cf895c32f119c0a1f1b 2024-11-16T05:39:52,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/b3efd0ca0aa5450db536f2df74331ae2 is 50, key is test_row_0/B:col10/1731735592135/Put/seqid=0 2024-11-16T05:39:52,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741903_1079 (size=12301) 2024-11-16T05:39:52,707 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/b3efd0ca0aa5450db536f2df74331ae2 2024-11-16T05:39:52,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/68b7c640c1374085b2f91343d261dfab is 50, key is test_row_0/C:col10/1731735592135/Put/seqid=0 2024-11-16T05:39:52,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741904_1080 (size=12301) 2024-11-16T05:39:52,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:52,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735652802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735652803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735652804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735652804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735652805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:39:52,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735652919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735652920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735652920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735652921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:52,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735652921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,069 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/6b7d6c09334a42d4a53ca441c055e4b4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/6b7d6c09334a42d4a53ca441c055e4b4 2024-11-16T05:39:53,078 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into 6b7d6c09334a42d4a53ca441c055e4b4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:53,078 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:53,079 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735592619; duration=0sec 2024-11-16T05:39:53,079 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:53,079 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:53,079 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:53,083 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:53,083 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:53,083 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:53,084 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/2096c60c4da04725b704d2e812b16a63, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/6b50d90db79f4c6a8189b0febee877c4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81912b1b9e6d46c08654c0b183a92001] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.5 K 2024-11-16T05:39:53,085 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2096c60c4da04725b704d2e812b16a63, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731735589645 2024-11-16T05:39:53,085 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b50d90db79f4c6a8189b0febee877c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731735590821 2024-11-16T05:39:53,086 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81912b1b9e6d46c08654c0b183a92001, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731735591476 2024-11-16T05:39:53,090 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/449dd95578154edda67c11a449fea073 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/449dd95578154edda67c11a449fea073 2024-11-16T05:39:53,098 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into 449dd95578154edda67c11a449fea073(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:53,100 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:53,100 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735592620; duration=0sec 2024-11-16T05:39:53,100 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:53,100 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:53,104 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:53,105 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/5a15d4ec0a534b9a89c9be1a5b473729 is 50, key is test_row_0/C:col10/1731735591476/Put/seqid=0 2024-11-16T05:39:53,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741905_1081 (size=13017) 2024-11-16T05:39:53,129 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/68b7c640c1374085b2f91343d261dfab 2024-11-16T05:39:53,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735653128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735653128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735653128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735653129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735653129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/722a56f90c154cf895c32f119c0a1f1b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/722a56f90c154cf895c32f119c0a1f1b 2024-11-16T05:39:53,139 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/722a56f90c154cf895c32f119c0a1f1b, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T05:39:53,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/b3efd0ca0aa5450db536f2df74331ae2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/b3efd0ca0aa5450db536f2df74331ae2 2024-11-16T05:39:53,147 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/b3efd0ca0aa5450db536f2df74331ae2, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T05:39:53,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/68b7c640c1374085b2f91343d261dfab as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/68b7c640c1374085b2f91343d261dfab 2024-11-16T05:39:53,152 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/68b7c640c1374085b2f91343d261dfab, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T05:39:53,154 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3992ee2db1edacc4ba76f5d2411f31eb in 482ms, sequenceid=311, compaction requested=false 2024-11-16T05:39:53,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:53,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:53,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-16T05:39:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-16T05:39:53,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-16T05:39:53,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 945 msec 2024-11-16T05:39:53,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 951 msec 2024-11-16T05:39:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:39:53,348 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:39:53,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:39:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-16T05:39:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:39:53,351 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:39:53,351 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:39:53,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:39:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:53,442 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:53,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T05:39:53,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/150e05b6e9da4752be5c64c6cbe2b53b is 50, key is test_row_0/A:col10/1731735592802/Put/seqid=0 2024-11-16T05:39:53,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741906_1082 (size=14741) 2024-11-16T05:39:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:39:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735653455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735653456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735653461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735653463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735653463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:39:53,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:53,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:53,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:53,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,520 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/5a15d4ec0a534b9a89c9be1a5b473729 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5a15d4ec0a534b9a89c9be1a5b473729 2024-11-16T05:39:53,527 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into 5a15d4ec0a534b9a89c9be1a5b473729(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:53,527 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:53,527 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735592620; duration=0sec 2024-11-16T05:39:53,527 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:53,527 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:53,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735653568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735653568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735653568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735653579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735653579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:39:53,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:53,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:53,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:53,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:39:53,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735653778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735653779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735653779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735653788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735653788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:53,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:39:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:53,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/150e05b6e9da4752be5c64c6cbe2b53b 2024-11-16T05:39:53,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/49aa8d3b3de24b84ac34ad3cbba65c62 is 50, key is test_row_0/B:col10/1731735592802/Put/seqid=0 2024-11-16T05:39:53,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741907_1083 (size=12301) 2024-11-16T05:39:53,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/49aa8d3b3de24b84ac34ad3cbba65c62 2024-11-16T05:39:53,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/91875ec6d5df47558f97eebce9826e0d is 50, key is test_row_0/C:col10/1731735592802/Put/seqid=0 2024-11-16T05:39:53,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741908_1084 (size=12301) 2024-11-16T05:39:53,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:39:53,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:53,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:53,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:53,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:39:54,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735654088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735654088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735654088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735654099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735654099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:39:54,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:54,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:54,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:54,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:54,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:54,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:39:54,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:54,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:54,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:54,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:54,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/91875ec6d5df47558f97eebce9826e0d 2024-11-16T05:39:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/150e05b6e9da4752be5c64c6cbe2b53b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/150e05b6e9da4752be5c64c6cbe2b53b 2024-11-16T05:39:54,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/150e05b6e9da4752be5c64c6cbe2b53b, entries=200, sequenceid=337, filesize=14.4 K 2024-11-16T05:39:54,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/49aa8d3b3de24b84ac34ad3cbba65c62 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/49aa8d3b3de24b84ac34ad3cbba65c62 2024-11-16T05:39:54,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/49aa8d3b3de24b84ac34ad3cbba65c62, entries=150, sequenceid=337, filesize=12.0 K 2024-11-16T05:39:54,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/91875ec6d5df47558f97eebce9826e0d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/91875ec6d5df47558f97eebce9826e0d 2024-11-16T05:39:54,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/91875ec6d5df47558f97eebce9826e0d, entries=150, sequenceid=337, filesize=12.0 K 2024-11-16T05:39:54,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 3992ee2db1edacc4ba76f5d2411f31eb in 884ms, sequenceid=337, compaction requested=true 2024-11-16T05:39:54,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:54,327 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:54,327 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:54,328 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:54,328 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:54,328 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:54,328 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:54,328 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:54,328 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:54,328 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/449dd95578154edda67c11a449fea073, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/b3efd0ca0aa5450db536f2df74331ae2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/49aa8d3b3de24b84ac34ad3cbba65c62] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.7 K 2024-11-16T05:39:54,328 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/6b7d6c09334a42d4a53ca441c055e4b4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/722a56f90c154cf895c32f119c0a1f1b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/150e05b6e9da4752be5c64c6cbe2b53b] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=39.1 K 2024-11-16T05:39:54,330 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b7d6c09334a42d4a53ca441c055e4b4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731735591476 2024-11-16T05:39:54,330 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 449dd95578154edda67c11a449fea073, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731735591476 2024-11-16T05:39:54,330 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 722a56f90c154cf895c32f119c0a1f1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731735592135 2024-11-16T05:39:54,330 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting b3efd0ca0aa5450db536f2df74331ae2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731735592135 2024-11-16T05:39:54,331 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 150e05b6e9da4752be5c64c6cbe2b53b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731735592802 2024-11-16T05:39:54,331 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 49aa8d3b3de24b84ac34ad3cbba65c62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731735592802 2024-11-16T05:39:54,353 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:54,353 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/c83346d322ee4a7cb1b3e935bc86b1c6 is 50, key is test_row_0/B:col10/1731735592802/Put/seqid=0 2024-11-16T05:39:54,355 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:54,356 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/2ab2142236734040a2ff6f8b60127129 is 50, key is test_row_0/A:col10/1731735592802/Put/seqid=0 2024-11-16T05:39:54,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741909_1085 (size=13119) 2024-11-16T05:39:54,378 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/2ab2142236734040a2ff6f8b60127129 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/2ab2142236734040a2ff6f8b60127129 2024-11-16T05:39:54,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741910_1086 (size=13119) 2024-11-16T05:39:54,386 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into 2ab2142236734040a2ff6f8b60127129(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:54,386 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:54,386 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735594326; duration=0sec 2024-11-16T05:39:54,387 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:54,387 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:54,387 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:54,389 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:54,389 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:54,389 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:54,389 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5a15d4ec0a534b9a89c9be1a5b473729, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/68b7c640c1374085b2f91343d261dfab, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/91875ec6d5df47558f97eebce9826e0d] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.7 K 2024-11-16T05:39:54,390 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a15d4ec0a534b9a89c9be1a5b473729, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731735591476 2024-11-16T05:39:54,390 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68b7c640c1374085b2f91343d261dfab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731735592135 2024-11-16T05:39:54,390 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91875ec6d5df47558f97eebce9826e0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731735592802 2024-11-16T05:39:54,400 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:54,401 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/d29fe685329e44018ec6dc49563ef4c9 is 50, key is test_row_0/C:col10/1731735592802/Put/seqid=0 2024-11-16T05:39:54,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741911_1087 (size=13119) 2024-11-16T05:39:54,417 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/d29fe685329e44018ec6dc49563ef4c9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/d29fe685329e44018ec6dc49563ef4c9 2024-11-16T05:39:54,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:39:54,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:54,425 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into d29fe685329e44018ec6dc49563ef4c9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:54,425 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:54,425 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735594327; duration=0sec 2024-11-16T05:39:54,425 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T05:39:54,425 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:54,425 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:54,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b0c59da645cd408195bb3bb5f8f17df8 is 50, key is test_row_0/A:col10/1731735593461/Put/seqid=0 2024-11-16T05:39:54,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741912_1088 (size=12301) 2024-11-16T05:39:54,436 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b0c59da645cd408195bb3bb5f8f17df8 2024-11-16T05:39:54,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/08ebaa214bde43d2a511396e42c292e5 is 50, key is test_row_0/B:col10/1731735593461/Put/seqid=0 2024-11-16T05:39:54,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741913_1089 (size=12301) 2024-11-16T05:39:54,456 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/08ebaa214bde43d2a511396e42c292e5 2024-11-16T05:39:54,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/5e60c5d7adc642a78c96a973aab55268 is 50, key is test_row_0/C:col10/1731735593461/Put/seqid=0 2024-11-16T05:39:54,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741914_1090 (size=12301) 2024-11-16T05:39:54,486 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/5e60c5d7adc642a78c96a973aab55268 2024-11-16T05:39:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:39:54,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b0c59da645cd408195bb3bb5f8f17df8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0c59da645cd408195bb3bb5f8f17df8 2024-11-16T05:39:54,504 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0c59da645cd408195bb3bb5f8f17df8, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T05:39:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/08ebaa214bde43d2a511396e42c292e5 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/08ebaa214bde43d2a511396e42c292e5 2024-11-16T05:39:54,518 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/08ebaa214bde43d2a511396e42c292e5, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T05:39:54,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/5e60c5d7adc642a78c96a973aab55268 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5e60c5d7adc642a78c96a973aab55268 2024-11-16T05:39:54,529 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5e60c5d7adc642a78c96a973aab55268, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T05:39:54,531 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 3992ee2db1edacc4ba76f5d2411f31eb in 106ms, sequenceid=353, compaction requested=false 2024-11-16T05:39:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-16T05:39:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-16T05:39:54,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-16T05:39:54,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1810 sec 2024-11-16T05:39:54,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1880 sec 2024-11-16T05:39:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:54,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:54,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:39:54,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b5d58e63463b4704b62deb5d0d7b7821 is 50, key is test_row_0/A:col10/1731735594605/Put/seqid=0 2024-11-16T05:39:54,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741915_1091 (size=12301) 2024-11-16T05:39:54,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735654673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735654675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735654675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735654676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735654678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735654788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735654788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,794 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/c83346d322ee4a7cb1b3e935bc86b1c6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/c83346d322ee4a7cb1b3e935bc86b1c6 2024-11-16T05:39:54,802 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into c83346d322ee4a7cb1b3e935bc86b1c6(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:54,802 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:54,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,802 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735594327; duration=0sec 2024-11-16T05:39:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735654800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,802 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:54,802 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:54,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735654800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735654800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735654998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735654998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735655008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735655008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735655008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b5d58e63463b4704b62deb5d0d7b7821 2024-11-16T05:39:55,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/4ad6bf36ddd2496ea788bb437ebce0ce is 50, key is test_row_0/B:col10/1731735594605/Put/seqid=0 2024-11-16T05:39:55,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741916_1092 (size=12301) 2024-11-16T05:39:55,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735655308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735655308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735655318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735655318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735655318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/4ad6bf36ddd2496ea788bb437ebce0ce 2024-11-16T05:39:55,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/7a538c9b9620477ba3e3886107dd52e2 is 50, key is test_row_0/C:col10/1731735594605/Put/seqid=0 2024-11-16T05:39:55,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741917_1093 (size=12301) 2024-11-16T05:39:55,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/7a538c9b9620477ba3e3886107dd52e2 2024-11-16T05:39:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:39:55,509 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:39:55,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:39:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-16T05:39:55,512 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:39:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:39:55,514 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:39:55,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:39:55,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b5d58e63463b4704b62deb5d0d7b7821 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b5d58e63463b4704b62deb5d0d7b7821 2024-11-16T05:39:55,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b5d58e63463b4704b62deb5d0d7b7821, entries=150, sequenceid=364, filesize=12.0 K 2024-11-16T05:39:55,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/4ad6bf36ddd2496ea788bb437ebce0ce as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4ad6bf36ddd2496ea788bb437ebce0ce 2024-11-16T05:39:55,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4ad6bf36ddd2496ea788bb437ebce0ce, entries=150, sequenceid=364, filesize=12.0 K 2024-11-16T05:39:55,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/7a538c9b9620477ba3e3886107dd52e2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7a538c9b9620477ba3e3886107dd52e2 2024-11-16T05:39:55,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7a538c9b9620477ba3e3886107dd52e2, entries=150, sequenceid=364, filesize=12.0 K 2024-11-16T05:39:55,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3992ee2db1edacc4ba76f5d2411f31eb in 956ms, sequenceid=364, compaction requested=true 2024-11-16T05:39:55,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:55,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:55,570 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:55,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:55,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:55,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:55,570 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:55,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:55,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:55,572 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:55,572 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:55,572 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:55,572 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/2ab2142236734040a2ff6f8b60127129, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0c59da645cd408195bb3bb5f8f17df8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b5d58e63463b4704b62deb5d0d7b7821] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.8 K 2024-11-16T05:39:55,572 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:55,572 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:55,572 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:55,572 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/c83346d322ee4a7cb1b3e935bc86b1c6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/08ebaa214bde43d2a511396e42c292e5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4ad6bf36ddd2496ea788bb437ebce0ce] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.8 K 2024-11-16T05:39:55,573 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ab2142236734040a2ff6f8b60127129, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731735592802 2024-11-16T05:39:55,573 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting c83346d322ee4a7cb1b3e935bc86b1c6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731735592802 2024-11-16T05:39:55,573 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c59da645cd408195bb3bb5f8f17df8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731735593455 2024-11-16T05:39:55,573 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ebaa214bde43d2a511396e42c292e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731735593455 2024-11-16T05:39:55,574 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ad6bf36ddd2496ea788bb437ebce0ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731735594605 2024-11-16T05:39:55,574 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5d58e63463b4704b62deb5d0d7b7821, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731735594605 2024-11-16T05:39:55,599 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:55,599 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:55,600 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/61c5eba1cb4e4a43ad0a094556aadb46 is 50, key is test_row_0/A:col10/1731735594605/Put/seqid=0 2024-11-16T05:39:55,600 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/3ca0e8a091c74801b1b80d90da791614 is 50, key is test_row_0/B:col10/1731735594605/Put/seqid=0 2024-11-16T05:39:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:39:55,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741918_1094 (size=13221) 2024-11-16T05:39:55,652 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/3ca0e8a091c74801b1b80d90da791614 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3ca0e8a091c74801b1b80d90da791614 2024-11-16T05:39:55,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741919_1095 (size=13221) 2024-11-16T05:39:55,665 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into 3ca0e8a091c74801b1b80d90da791614(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:55,665 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:55,665 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735595570; duration=0sec 2024-11-16T05:39:55,666 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:55,666 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:55,666 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:55,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T05:39:55,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:55,668 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T05:39:55,670 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:55,670 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:55,670 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:55,670 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/d29fe685329e44018ec6dc49563ef4c9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5e60c5d7adc642a78c96a973aab55268, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7a538c9b9620477ba3e3886107dd52e2] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.8 K 2024-11-16T05:39:55,671 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting d29fe685329e44018ec6dc49563ef4c9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731735592802 2024-11-16T05:39:55,672 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e60c5d7adc642a78c96a973aab55268, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731735593455 2024-11-16T05:39:55,672 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a538c9b9620477ba3e3886107dd52e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731735594605 2024-11-16T05:39:55,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/0f0eae2a31bb49efab4a1c73e2de4750 is 50, key is test_row_0/A:col10/1731735594674/Put/seqid=0 2024-11-16T05:39:55,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741920_1096 (size=12301) 2024-11-16T05:39:55,704 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/0f0eae2a31bb49efab4a1c73e2de4750 2024-11-16T05:39:55,707 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:55,708 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/81efb738e3f14690a5bd6f4a7848a540 is 50, key is test_row_0/C:col10/1731735594605/Put/seqid=0 2024-11-16T05:39:55,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/6639610573e34ed0a436779779e9ba5e is 50, key is test_row_0/B:col10/1731735594674/Put/seqid=0 2024-11-16T05:39:55,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741921_1097 (size=13221) 2024-11-16T05:39:55,751 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/81efb738e3f14690a5bd6f4a7848a540 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81efb738e3f14690a5bd6f4a7848a540 2024-11-16T05:39:55,761 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into 81efb738e3f14690a5bd6f4a7848a540(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:55,761 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:55,762 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735595570; duration=0sec 2024-11-16T05:39:55,762 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:55,762 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:55,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741922_1098 (size=12301) 2024-11-16T05:39:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:55,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:39:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735655834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735655835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735655836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735655837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735655837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735655948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735655948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735655948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735655948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735655948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,071 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/61c5eba1cb4e4a43ad0a094556aadb46 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/61c5eba1cb4e4a43ad0a094556aadb46 2024-11-16T05:39:56,083 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into 61c5eba1cb4e4a43ad0a094556aadb46(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:56,083 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:56,083 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735595570; duration=0sec 2024-11-16T05:39:56,083 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:56,083 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:39:56,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735656158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735656158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735656158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735656158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735656158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,176 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/6639610573e34ed0a436779779e9ba5e 2024-11-16T05:39:56,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/feb3365cef7640b1b3b1572bd5aeb742 is 50, key is test_row_0/C:col10/1731735594674/Put/seqid=0 2024-11-16T05:39:56,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741923_1099 (size=12301) 2024-11-16T05:39:56,200 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/feb3365cef7640b1b3b1572bd5aeb742 2024-11-16T05:39:56,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/0f0eae2a31bb49efab4a1c73e2de4750 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/0f0eae2a31bb49efab4a1c73e2de4750 2024-11-16T05:39:56,212 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/0f0eae2a31bb49efab4a1c73e2de4750, entries=150, sequenceid=391, filesize=12.0 K 2024-11-16T05:39:56,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/6639610573e34ed0a436779779e9ba5e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6639610573e34ed0a436779779e9ba5e 2024-11-16T05:39:56,222 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6639610573e34ed0a436779779e9ba5e, entries=150, sequenceid=391, filesize=12.0 K 2024-11-16T05:39:56,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/feb3365cef7640b1b3b1572bd5aeb742 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/feb3365cef7640b1b3b1572bd5aeb742 2024-11-16T05:39:56,231 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/feb3365cef7640b1b3b1572bd5aeb742, entries=150, sequenceid=391, filesize=12.0 K 2024-11-16T05:39:56,232 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 3992ee2db1edacc4ba76f5d2411f31eb in 565ms, sequenceid=391, compaction requested=false 2024-11-16T05:39:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-16T05:39:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-16T05:39:56,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-16T05:39:56,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 722 msec 2024-11-16T05:39:56,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 730 msec 2024-11-16T05:39:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:56,470 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:56,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T05:39:56,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/a103b66b57ca409ab14bf411c38da3f0 is 50, key is test_row_0/A:col10/1731735595835/Put/seqid=0 2024-11-16T05:39:56,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741924_1100 (size=14741) 2024-11-16T05:39:56,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735656507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735656509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735656509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735656511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735656511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735656618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735656618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735656618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735656619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735656619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:39:56,649 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:39:56,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:39:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-16T05:39:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:39:56,652 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:39:56,653 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:39:56,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:39:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:39:56,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T05:39:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:56,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735656828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735656828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735656828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735656828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735656828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:56,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:56,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:56,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/a103b66b57ca409ab14bf411c38da3f0 2024-11-16T05:39:56,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/51c871147452410da79f9d89ef9c702b is 50, key is test_row_0/B:col10/1731735595835/Put/seqid=0 2024-11-16T05:39:56,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741925_1101 (size=12301) 2024-11-16T05:39:56,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T05:39:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:56,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:56,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:56,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:56,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:39:57,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T05:39:57,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:57,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:57,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:57,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:57,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:57,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735657138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735657138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735657139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735657139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735657139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T05:39:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:57,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:39:57,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/51c871147452410da79f9d89ef9c702b 2024-11-16T05:39:57,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/9128dd03eea64d15be15b7a3b23c2007 is 50, key is test_row_0/C:col10/1731735595835/Put/seqid=0 2024-11-16T05:39:57,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741926_1102 (size=12301) 2024-11-16T05:39:57,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/9128dd03eea64d15be15b7a3b23c2007 2024-11-16T05:39:57,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/a103b66b57ca409ab14bf411c38da3f0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a103b66b57ca409ab14bf411c38da3f0 2024-11-16T05:39:57,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a103b66b57ca409ab14bf411c38da3f0, entries=200, sequenceid=407, filesize=14.4 K 2024-11-16T05:39:57,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/51c871147452410da79f9d89ef9c702b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/51c871147452410da79f9d89ef9c702b 2024-11-16T05:39:57,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/51c871147452410da79f9d89ef9c702b, entries=150, sequenceid=407, filesize=12.0 K 2024-11-16T05:39:57,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/9128dd03eea64d15be15b7a3b23c2007 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/9128dd03eea64d15be15b7a3b23c2007 2024-11-16T05:39:57,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/9128dd03eea64d15be15b7a3b23c2007, entries=150, sequenceid=407, filesize=12.0 K 2024-11-16T05:39:57,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3992ee2db1edacc4ba76f5d2411f31eb in 936ms, sequenceid=407, compaction requested=true 2024-11-16T05:39:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:57,406 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:57,406 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:57,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:57,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:57,407 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:57,407 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:57,408 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:57,408 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:57,408 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:57,408 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3ca0e8a091c74801b1b80d90da791614, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6639610573e34ed0a436779779e9ba5e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/51c871147452410da79f9d89ef9c702b] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.9 K 2024-11-16T05:39:57,408 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:57,408 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/61c5eba1cb4e4a43ad0a094556aadb46, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/0f0eae2a31bb49efab4a1c73e2de4750, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a103b66b57ca409ab14bf411c38da3f0] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=39.3 K 2024-11-16T05:39:57,409 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ca0e8a091c74801b1b80d90da791614, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731735594605 2024-11-16T05:39:57,409 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61c5eba1cb4e4a43ad0a094556aadb46, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731735594605 2024-11-16T05:39:57,410 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f0eae2a31bb49efab4a1c73e2de4750, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731735594655 2024-11-16T05:39:57,410 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6639610573e34ed0a436779779e9ba5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731735594655 2024-11-16T05:39:57,410 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a103b66b57ca409ab14bf411c38da3f0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731735595833 2024-11-16T05:39:57,410 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c871147452410da79f9d89ef9c702b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731735595833 2024-11-16T05:39:57,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T05:39:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:57,421 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T05:39:57,425 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:57,426 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/2e285cee25a04ed9870b38e08c9429a0 is 50, key is test_row_0/B:col10/1731735595835/Put/seqid=0 2024-11-16T05:39:57,426 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:57,427 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/c156f9303c014b5280d133971fc5b6d6 is 50, key is test_row_0/A:col10/1731735595835/Put/seqid=0 2024-11-16T05:39:57,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/ab628ea0c995412e993af2358b1d5ad0 is 50, key is test_row_0/A:col10/1731735596509/Put/seqid=0 2024-11-16T05:39:57,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741927_1103 (size=13323) 2024-11-16T05:39:57,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741928_1104 (size=13323) 2024-11-16T05:39:57,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741929_1105 (size=12301) 2024-11-16T05:39:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:57,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:57,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735657661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735657662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735657663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735657665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735657666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735657769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735657769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735657778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735657779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735657779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:39:57,862 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/ab628ea0c995412e993af2358b1d5ad0 2024-11-16T05:39:57,862 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/2e285cee25a04ed9870b38e08c9429a0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2e285cee25a04ed9870b38e08c9429a0 2024-11-16T05:39:57,862 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/c156f9303c014b5280d133971fc5b6d6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/c156f9303c014b5280d133971fc5b6d6 2024-11-16T05:39:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/695802fa19f74553b7d9eb3cc0480fe8 is 50, key is test_row_0/B:col10/1731735596509/Put/seqid=0 2024-11-16T05:39:57,883 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into 2e285cee25a04ed9870b38e08c9429a0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:57,883 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:57,883 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735597406; duration=0sec 2024-11-16T05:39:57,883 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:57,883 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:57,884 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:57,884 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into c156f9303c014b5280d133971fc5b6d6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:57,884 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:57,884 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735597406; duration=0sec 2024-11-16T05:39:57,884 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:57,884 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:57,886 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:57,886 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:57,886 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:57,886 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81efb738e3f14690a5bd6f4a7848a540, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/feb3365cef7640b1b3b1572bd5aeb742, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/9128dd03eea64d15be15b7a3b23c2007] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=36.9 K 2024-11-16T05:39:57,887 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 81efb738e3f14690a5bd6f4a7848a540, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731735594605 2024-11-16T05:39:57,888 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting feb3365cef7640b1b3b1572bd5aeb742, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731735594655 2024-11-16T05:39:57,888 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9128dd03eea64d15be15b7a3b23c2007, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731735595833 2024-11-16T05:39:57,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741930_1106 (size=12301) 2024-11-16T05:39:57,907 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:57,907 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/ebc35b139d7b495e9813d5de0b26bab7 is 50, key is test_row_0/C:col10/1731735595835/Put/seqid=0 2024-11-16T05:39:57,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741931_1107 (size=13323) 2024-11-16T05:39:57,930 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/ebc35b139d7b495e9813d5de0b26bab7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ebc35b139d7b495e9813d5de0b26bab7 2024-11-16T05:39:57,937 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into ebc35b139d7b495e9813d5de0b26bab7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:57,937 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:57,937 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735597406; duration=0sec 2024-11-16T05:39:57,937 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:57,937 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735657978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735657978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735657988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735657989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:57,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735657989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735658288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735658288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735658299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735658299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,300 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/695802fa19f74553b7d9eb3cc0480fe8 2024-11-16T05:39:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735658299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/dcb92e8bba334cc58c209ede187fa17e is 50, key is test_row_0/C:col10/1731735596509/Put/seqid=0 2024-11-16T05:39:58,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741932_1108 (size=12301) 2024-11-16T05:39:58,715 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/dcb92e8bba334cc58c209ede187fa17e 2024-11-16T05:39:58,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/ab628ea0c995412e993af2358b1d5ad0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ab628ea0c995412e993af2358b1d5ad0 2024-11-16T05:39:58,733 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ab628ea0c995412e993af2358b1d5ad0, entries=150, sequenceid=429, filesize=12.0 K 2024-11-16T05:39:58,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/695802fa19f74553b7d9eb3cc0480fe8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/695802fa19f74553b7d9eb3cc0480fe8 2024-11-16T05:39:58,751 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/695802fa19f74553b7d9eb3cc0480fe8, entries=150, sequenceid=429, filesize=12.0 K 2024-11-16T05:39:58,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/dcb92e8bba334cc58c209ede187fa17e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/dcb92e8bba334cc58c209ede187fa17e 2024-11-16T05:39:58,764 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/dcb92e8bba334cc58c209ede187fa17e, entries=150, sequenceid=429, filesize=12.0 K 2024-11-16T05:39:58,767 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3992ee2db1edacc4ba76f5d2411f31eb in 1347ms, sequenceid=429, compaction requested=false 2024-11-16T05:39:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-16T05:39:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-16T05:39:58,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-16T05:39:58,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1180 sec 2024-11-16T05:39:58,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1270 sec 2024-11-16T05:39:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:39:58,799 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:39:58,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:39:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-16T05:39:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T05:39:58,802 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:39:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:58,803 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:39:58,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T05:39:58,804 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:39:58,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:39:58,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b0d9f1224c91458b8693f7764f1955fd is 50, key is test_row_0/A:col10/1731735598800/Put/seqid=0 2024-11-16T05:39:58,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741933_1109 (size=17181) 2024-11-16T05:39:58,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b0d9f1224c91458b8693f7764f1955fd 2024-11-16T05:39:58,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/34eff7e9e6ab45c3b6aedc5e2c32913d is 50, key is test_row_0/B:col10/1731735598800/Put/seqid=0 2024-11-16T05:39:58,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735658834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735658833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735658837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741934_1110 (size=12301) 2024-11-16T05:39:58,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735658842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735658843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/34eff7e9e6ab45c3b6aedc5e2c32913d 2024-11-16T05:39:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/92f6e740cf8d4e4e8baaa5bc34bfc852 is 50, key is test_row_0/C:col10/1731735598800/Put/seqid=0 2024-11-16T05:39:58,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741935_1111 (size=12301) 2024-11-16T05:39:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T05:39:58,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735658948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735658949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735658949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T05:39:58,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:58,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:58,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:58,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:58,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:58,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735658958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735658958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:58,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T05:39:59,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:59,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:59,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:59,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:59,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T05:39:59,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735659160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735659160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735659160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735659168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735659168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T05:39:59,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:59,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:59,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:59,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:59,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:39:59,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/92f6e740cf8d4e4e8baaa5bc34bfc852 2024-11-16T05:39:59,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/b0d9f1224c91458b8693f7764f1955fd as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0d9f1224c91458b8693f7764f1955fd 2024-11-16T05:39:59,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0d9f1224c91458b8693f7764f1955fd, entries=250, sequenceid=448, filesize=16.8 K 2024-11-16T05:39:59,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/34eff7e9e6ab45c3b6aedc5e2c32913d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/34eff7e9e6ab45c3b6aedc5e2c32913d 2024-11-16T05:39:59,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/34eff7e9e6ab45c3b6aedc5e2c32913d, entries=150, sequenceid=448, filesize=12.0 K 2024-11-16T05:39:59,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/92f6e740cf8d4e4e8baaa5bc34bfc852 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/92f6e740cf8d4e4e8baaa5bc34bfc852 2024-11-16T05:39:59,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/92f6e740cf8d4e4e8baaa5bc34bfc852, entries=150, sequenceid=448, filesize=12.0 K 2024-11-16T05:39:59,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3992ee2db1edacc4ba76f5d2411f31eb in 501ms, sequenceid=448, compaction requested=true 2024-11-16T05:39:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:39:59,305 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:59,305 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:39:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:39:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:59,306 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:59,307 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:39:59,307 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:59,307 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/c156f9303c014b5280d133971fc5b6d6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ab628ea0c995412e993af2358b1d5ad0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0d9f1224c91458b8693f7764f1955fd] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=41.8 K 2024-11-16T05:39:59,307 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:59,307 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:39:59,308 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:59,308 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2e285cee25a04ed9870b38e08c9429a0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/695802fa19f74553b7d9eb3cc0480fe8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/34eff7e9e6ab45c3b6aedc5e2c32913d] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=37.0 K 2024-11-16T05:39:59,308 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c156f9303c014b5280d133971fc5b6d6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731735595833 2024-11-16T05:39:59,308 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e285cee25a04ed9870b38e08c9429a0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731735595833 2024-11-16T05:39:59,309 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab628ea0c995412e993af2358b1d5ad0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731735596493 2024-11-16T05:39:59,309 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 695802fa19f74553b7d9eb3cc0480fe8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731735596493 2024-11-16T05:39:59,310 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 34eff7e9e6ab45c3b6aedc5e2c32913d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731735597662 2024-11-16T05:39:59,310 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0d9f1224c91458b8693f7764f1955fd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731735597661 2024-11-16T05:39:59,325 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:59,326 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/831223de128040a2925eb4068b1cb9d1 is 50, key is test_row_0/A:col10/1731735598800/Put/seqid=0 2024-11-16T05:39:59,335 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:59,336 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/2bdd332649674b0ea102be6ccd87f850 is 50, key is test_row_0/B:col10/1731735598800/Put/seqid=0 2024-11-16T05:39:59,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741936_1112 (size=13425) 2024-11-16T05:39:59,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741937_1113 (size=13425) 2024-11-16T05:39:59,379 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/831223de128040a2925eb4068b1cb9d1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/831223de128040a2925eb4068b1cb9d1 2024-11-16T05:39:59,406 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into 831223de128040a2925eb4068b1cb9d1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:59,406 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:59,406 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735599305; duration=0sec 2024-11-16T05:39:59,408 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:39:59,408 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:39:59,408 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:39:59,410 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:39:59,410 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:39:59,410 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:59,410 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ebc35b139d7b495e9813d5de0b26bab7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/dcb92e8bba334cc58c209ede187fa17e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/92f6e740cf8d4e4e8baaa5bc34bfc852] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=37.0 K 2024-11-16T05:39:59,411 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebc35b139d7b495e9813d5de0b26bab7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731735595833 2024-11-16T05:39:59,411 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcb92e8bba334cc58c209ede187fa17e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731735596493 2024-11-16T05:39:59,412 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92f6e740cf8d4e4e8baaa5bc34bfc852, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731735597662 2024-11-16T05:39:59,422 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:39:59,423 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/4b836914d55e41b7ae5c3167dcd3a4b5 is 50, key is test_row_0/C:col10/1731735598800/Put/seqid=0 2024-11-16T05:39:59,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T05:39:59,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:59,423 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T05:39:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T05:39:59,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/272fd54da487415f9c7493277dcf2310 is 50, key is test_row_0/A:col10/1731735598839/Put/seqid=0 2024-11-16T05:39:59,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741938_1114 (size=13425) 2024-11-16T05:39:59,446 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/4b836914d55e41b7ae5c3167dcd3a4b5 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4b836914d55e41b7ae5c3167dcd3a4b5 2024-11-16T05:39:59,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741939_1115 (size=12301) 2024-11-16T05:39:59,455 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/272fd54da487415f9c7493277dcf2310 2024-11-16T05:39:59,457 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into 4b836914d55e41b7ae5c3167dcd3a4b5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:59,457 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:59,457 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735599305; duration=0sec 2024-11-16T05:39:59,457 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:59,457 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:39:59,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/a3c35aa43a41445c8a86a0ca6cb5bf29 is 50, key is test_row_0/B:col10/1731735598839/Put/seqid=0 2024-11-16T05:39:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:39:59,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:39:59,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735659498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741940_1116 (size=12301) 2024-11-16T05:39:59,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735659499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,508 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/a3c35aa43a41445c8a86a0ca6cb5bf29 2024-11-16T05:39:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735659501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735659503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735659503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/16ba291bdd594319a235fff9f96fa31c is 50, key is test_row_0/C:col10/1731735598839/Put/seqid=0 2024-11-16T05:39:59,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741941_1117 (size=12301) 2024-11-16T05:39:59,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735659609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735659618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735659618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735659618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735659618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,771 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/2bdd332649674b0ea102be6ccd87f850 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2bdd332649674b0ea102be6ccd87f850 2024-11-16T05:39:59,783 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into 2bdd332649674b0ea102be6ccd87f850(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:39:59,783 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:59,783 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735599305; duration=0sec 2024-11-16T05:39:59,783 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:39:59,783 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:39:59,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735659818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735659828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735659828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735659829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:39:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735659829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:39:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:39:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:39:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T05:39:59,939 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/16ba291bdd594319a235fff9f96fa31c 2024-11-16T05:39:59,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/272fd54da487415f9c7493277dcf2310 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/272fd54da487415f9c7493277dcf2310 2024-11-16T05:39:59,952 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/272fd54da487415f9c7493277dcf2310, entries=150, sequenceid=469, filesize=12.0 K 2024-11-16T05:39:59,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/a3c35aa43a41445c8a86a0ca6cb5bf29 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/a3c35aa43a41445c8a86a0ca6cb5bf29 2024-11-16T05:39:59,974 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/a3c35aa43a41445c8a86a0ca6cb5bf29, entries=150, sequenceid=469, filesize=12.0 K 2024-11-16T05:39:59,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/16ba291bdd594319a235fff9f96fa31c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/16ba291bdd594319a235fff9f96fa31c 2024-11-16T05:39:59,987 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/16ba291bdd594319a235fff9f96fa31c, entries=150, sequenceid=469, filesize=12.0 K 2024-11-16T05:39:59,989 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 3992ee2db1edacc4ba76f5d2411f31eb in 566ms, sequenceid=469, compaction requested=false 2024-11-16T05:39:59,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:39:59,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:39:59,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-16T05:39:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-16T05:39:59,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-16T05:39:59,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1880 sec 2024-11-16T05:39:59,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1950 sec 2024-11-16T05:40:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:40:00,132 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:00,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T05:40:00,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/4d3da80cd98f475a85288c082b8f18b4 is 50, key is test_row_0/A:col10/1731735599496/Put/seqid=0 2024-11-16T05:40:00,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741942_1118 (size=12301) 2024-11-16T05:40:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735660161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735660162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735660164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735660164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735660164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735660268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735660268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735660278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735660278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735660278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735660479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735660480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735660488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735660488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735660490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/4d3da80cd98f475a85288c082b8f18b4 2024-11-16T05:40:00,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/ea1e4f60fe964905b47e4fbf7dc3b028 is 50, key is test_row_0/B:col10/1731735599496/Put/seqid=0 2024-11-16T05:40:00,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741943_1119 (size=12301) 2024-11-16T05:40:00,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/ea1e4f60fe964905b47e4fbf7dc3b028 2024-11-16T05:40:00,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/60b85942cb144b9da841bccc3b76f04f is 50, key is test_row_0/C:col10/1731735599496/Put/seqid=0 2024-11-16T05:40:00,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741944_1120 (size=12301) 2024-11-16T05:40:00,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/60b85942cb144b9da841bccc3b76f04f 2024-11-16T05:40:00,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/4d3da80cd98f475a85288c082b8f18b4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4d3da80cd98f475a85288c082b8f18b4 2024-11-16T05:40:00,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4d3da80cd98f475a85288c082b8f18b4, entries=150, sequenceid=489, filesize=12.0 K 2024-11-16T05:40:00,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/ea1e4f60fe964905b47e4fbf7dc3b028 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ea1e4f60fe964905b47e4fbf7dc3b028 2024-11-16T05:40:00,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ea1e4f60fe964905b47e4fbf7dc3b028, entries=150, sequenceid=489, filesize=12.0 K 2024-11-16T05:40:00,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/60b85942cb144b9da841bccc3b76f04f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/60b85942cb144b9da841bccc3b76f04f 2024-11-16T05:40:00,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/60b85942cb144b9da841bccc3b76f04f, entries=150, sequenceid=489, filesize=12.0 K 2024-11-16T05:40:00,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 3992ee2db1edacc4ba76f5d2411f31eb in 595ms, sequenceid=489, compaction requested=true 2024-11-16T05:40:00,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:40:00,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:00,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:00,728 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:00,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:00,728 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:00,729 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:00,729 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:00,730 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/A is initiating minor compaction (all files) 2024-11-16T05:40:00,730 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/B is initiating minor compaction (all files) 2024-11-16T05:40:00,730 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/A in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:00,730 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/B in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:00,730 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/831223de128040a2925eb4068b1cb9d1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/272fd54da487415f9c7493277dcf2310, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4d3da80cd98f475a85288c082b8f18b4] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=37.1 K 2024-11-16T05:40:00,730 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2bdd332649674b0ea102be6ccd87f850, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/a3c35aa43a41445c8a86a0ca6cb5bf29, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ea1e4f60fe964905b47e4fbf7dc3b028] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=37.1 K 2024-11-16T05:40:00,730 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 831223de128040a2925eb4068b1cb9d1, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731735597662 2024-11-16T05:40:00,730 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bdd332649674b0ea102be6ccd87f850, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731735597662 2024-11-16T05:40:00,731 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting a3c35aa43a41445c8a86a0ca6cb5bf29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731735598832 2024-11-16T05:40:00,731 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 272fd54da487415f9c7493277dcf2310, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731735598832 2024-11-16T05:40:00,732 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d3da80cd98f475a85288c082b8f18b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731735599496 2024-11-16T05:40:00,732 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ea1e4f60fe964905b47e4fbf7dc3b028, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731735599496 2024-11-16T05:40:00,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:00,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3992ee2db1edacc4ba76f5d2411f31eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:00,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:00,746 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#A#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:00,746 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e93d3ac8fb724573959449c442bd1045 is 50, key is test_row_0/A:col10/1731735599496/Put/seqid=0 2024-11-16T05:40:00,755 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#B#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:00,756 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/89d4d3dabb404526a884aa3263d6010c is 50, key is test_row_0/B:col10/1731735599496/Put/seqid=0 2024-11-16T05:40:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:40:00,790 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:00,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T05:40:00,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741946_1122 (size=13527) 2024-11-16T05:40:00,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741945_1121 (size=13527) 2024-11-16T05:40:00,806 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e93d3ac8fb724573959449c442bd1045 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e93d3ac8fb724573959449c442bd1045 2024-11-16T05:40:00,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e2ff8f6efc9744879a3a0347e7edd1c4 is 50, key is test_row_0/A:col10/1731735600154/Put/seqid=0 2024-11-16T05:40:00,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735660818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735660818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735660822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735660829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735660834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,841 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/A of 3992ee2db1edacc4ba76f5d2411f31eb into e93d3ac8fb724573959449c442bd1045(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:00,841 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:40:00,841 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/A, priority=13, startTime=1731735600728; duration=0sec 2024-11-16T05:40:00,841 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:00,841 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:A 2024-11-16T05:40:00,841 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:00,842 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:00,843 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 3992ee2db1edacc4ba76f5d2411f31eb/C is initiating minor compaction (all files) 2024-11-16T05:40:00,843 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3992ee2db1edacc4ba76f5d2411f31eb/C in TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:00,843 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4b836914d55e41b7ae5c3167dcd3a4b5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/16ba291bdd594319a235fff9f96fa31c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/60b85942cb144b9da841bccc3b76f04f] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp, totalSize=37.1 K 2024-11-16T05:40:00,843 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b836914d55e41b7ae5c3167dcd3a4b5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731735597662 2024-11-16T05:40:00,844 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16ba291bdd594319a235fff9f96fa31c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731735598832 2024-11-16T05:40:00,844 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60b85942cb144b9da841bccc3b76f04f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731735599496 2024-11-16T05:40:00,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741947_1123 (size=17181) 2024-11-16T05:40:00,874 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3992ee2db1edacc4ba76f5d2411f31eb#C#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:00,875 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/c93959c049094791ab94dfa14d09e939 is 50, key is test_row_0/C:col10/1731735599496/Put/seqid=0 2024-11-16T05:40:00,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741948_1124 (size=13527) 2024-11-16T05:40:00,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735660939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735660939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735660939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T05:40:00,949 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:00,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735660949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735660949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-16T05:40:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T05:40:00,952 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:00,953 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:00,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T05:40:01,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T05:40:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:40:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:01,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:01,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:01,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731735661148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60894 deadline: 1731735661148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60916 deadline: 1731735661148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:01,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731735661159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:01,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60892 deadline: 1731735661159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:01,205 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/89d4d3dabb404526a884aa3263d6010c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/89d4d3dabb404526a884aa3263d6010c 2024-11-16T05:40:01,213 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/B of 3992ee2db1edacc4ba76f5d2411f31eb into 89d4d3dabb404526a884aa3263d6010c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:01,214 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:40:01,214 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/B, priority=13, startTime=1731735600728; duration=0sec 2024-11-16T05:40:01,214 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:01,214 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:B 2024-11-16T05:40:01,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T05:40:01,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:01,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:40:01,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:01,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:01,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:01,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e2ff8f6efc9744879a3a0347e7edd1c4 2024-11-16T05:40:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T05:40:01,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/f75bee6dd30b45a19254545113bc346f is 50, key is test_row_0/B:col10/1731735600154/Put/seqid=0 2024-11-16T05:40:01,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741949_1125 (size=12301) 2024-11-16T05:40:01,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/f75bee6dd30b45a19254545113bc346f 2024-11-16T05:40:01,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/b01906ba322f4ce793c365dcb3d7dc26 is 50, key is test_row_0/C:col10/1731735600154/Put/seqid=0 2024-11-16T05:40:01,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741950_1126 (size=12301) 2024-11-16T05:40:01,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/b01906ba322f4ce793c365dcb3d7dc26 2024-11-16T05:40:01,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e2ff8f6efc9744879a3a0347e7edd1c4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e2ff8f6efc9744879a3a0347e7edd1c4 2024-11-16T05:40:01,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e2ff8f6efc9744879a3a0347e7edd1c4, entries=250, sequenceid=509, filesize=16.8 K 2024-11-16T05:40:01,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/f75bee6dd30b45a19254545113bc346f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/f75bee6dd30b45a19254545113bc346f 2024-11-16T05:40:01,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/f75bee6dd30b45a19254545113bc346f, entries=150, sequenceid=509, filesize=12.0 K 2024-11-16T05:40:01,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/b01906ba322f4ce793c365dcb3d7dc26 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/b01906ba322f4ce793c365dcb3d7dc26 2024-11-16T05:40:01,346 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/c93959c049094791ab94dfa14d09e939 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c93959c049094791ab94dfa14d09e939 2024-11-16T05:40:01,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/b01906ba322f4ce793c365dcb3d7dc26, entries=150, sequenceid=509, filesize=12.0 K 2024-11-16T05:40:01,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3992ee2db1edacc4ba76f5d2411f31eb in 567ms, sequenceid=509, compaction requested=false 2024-11-16T05:40:01,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:40:01,359 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3992ee2db1edacc4ba76f5d2411f31eb/C of 3992ee2db1edacc4ba76f5d2411f31eb into c93959c049094791ab94dfa14d09e939(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:01,359 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:40:01,359 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., storeName=3992ee2db1edacc4ba76f5d2411f31eb/C, priority=13, startTime=1731735600733; duration=0sec 2024-11-16T05:40:01,359 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:01,359 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3992ee2db1edacc4ba76f5d2411f31eb:C 2024-11-16T05:40:01,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T05:40:01,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:01,415 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T05:40:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/1462f25ea75340f293bb49826cea63e9 is 50, key is test_row_0/A:col10/1731735600813/Put/seqid=0 2024-11-16T05:40:01,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741951_1127 (size=12301) 2024-11-16T05:40:01,438 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-16T05:40:01,438 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-16T05:40:01,438 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:01,438 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:01,439 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-16T05:40:01,439 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:01,441 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-16T05:40:01,441 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:01,442 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,442 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,442 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,442 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,443 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,443 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:01,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:01,443 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:01,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:01,444 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:40:01,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. as already flushing 2024-11-16T05:40:01,460 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-16T05:40:01,460 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-16T05:40:01,460 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-16T05:40:01,460 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:01,460 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:01,460 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,460 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:01,460 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,460 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,460 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,460 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,460 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:01,460 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:01,460 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,460 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:01,471 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-16T05:40:01,471 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-16T05:40:01,471 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:01,471 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:01,471 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,471 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,471 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,471 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:01,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:01,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T05:40:01,825 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/1462f25ea75340f293bb49826cea63e9 2024-11-16T05:40:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/1caf9e9d25fc4140986d8a1129f0d9e9 is 50, key is test_row_0/B:col10/1731735600813/Put/seqid=0 2024-11-16T05:40:01,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741952_1128 (size=12301) 2024-11-16T05:40:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T05:40:02,243 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/1caf9e9d25fc4140986d8a1129f0d9e9 2024-11-16T05:40:02,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/d70d6998c57f46998c5602dea01cab5c is 50, key is test_row_0/C:col10/1731735600813/Put/seqid=0 2024-11-16T05:40:02,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741953_1129 (size=12301) 2024-11-16T05:40:02,662 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/d70d6998c57f46998c5602dea01cab5c 2024-11-16T05:40:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/1462f25ea75340f293bb49826cea63e9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/1462f25ea75340f293bb49826cea63e9 2024-11-16T05:40:02,682 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/1462f25ea75340f293bb49826cea63e9, entries=150, sequenceid=528, filesize=12.0 K 2024-11-16T05:40:02,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/1caf9e9d25fc4140986d8a1129f0d9e9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/1caf9e9d25fc4140986d8a1129f0d9e9 2024-11-16T05:40:02,690 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/1caf9e9d25fc4140986d8a1129f0d9e9, entries=150, sequenceid=528, filesize=12.0 K 2024-11-16T05:40:02,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/d70d6998c57f46998c5602dea01cab5c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/d70d6998c57f46998c5602dea01cab5c 2024-11-16T05:40:02,697 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/d70d6998c57f46998c5602dea01cab5c, entries=150, sequenceid=528, filesize=12.0 K 2024-11-16T05:40:02,698 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=33.54 KB/34350 for 3992ee2db1edacc4ba76f5d2411f31eb in 1284ms, sequenceid=528, compaction requested=true 2024-11-16T05:40:02,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 3992ee2db1edacc4ba76f5d2411f31eb: 2024-11-16T05:40:02,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:02,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-16T05:40:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-16T05:40:02,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-16T05:40:02,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7470 sec 2024-11-16T05:40:02,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.7540 sec 2024-11-16T05:40:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T05:40:03,099 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:03,099 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3284 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3227 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1488 2024-11-16T05:40:03,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4462 rows 2024-11-16T05:40:03,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1482 2024-11-16T05:40:03,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4443 rows 2024-11-16T05:40:03,101 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:40:03,101 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:40:03,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:03,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:03,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:03,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:40:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T05:40:03,115 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735603115"}]},"ts":"1731735603115"} 2024-11-16T05:40:03,117 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:40:03,117 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:40:03,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:40:03,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3992ee2db1edacc4ba76f5d2411f31eb, UNASSIGN}] 2024-11-16T05:40:03,125 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3992ee2db1edacc4ba76f5d2411f31eb, UNASSIGN 2024-11-16T05:40:03,127 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3992ee2db1edacc4ba76f5d2411f31eb, regionState=CLOSING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:03,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3992ee2db1edacc4ba76f5d2411f31eb, UNASSIGN because future has completed 2024-11-16T05:40:03,132 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:40:03,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:40:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T05:40:03,290 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:40:03,290 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:40:03,291 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 3992ee2db1edacc4ba76f5d2411f31eb, disabling compactions & flushes 2024-11-16T05:40:03,291 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:03,291 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:03,291 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. after waiting 0 ms 2024-11-16T05:40:03,291 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:03,291 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 3992ee2db1edacc4ba76f5d2411f31eb 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T05:40:03,298 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e81200c052314a8f9cd708142e689e8f is 50, key is test_row_0/A:col10/1731735601469/Put/seqid=0 2024-11-16T05:40:03,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741954_1130 (size=12301) 2024-11-16T05:40:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T05:40:03,555 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:40:03,705 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e81200c052314a8f9cd708142e689e8f 2024-11-16T05:40:03,718 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/f38d1db4f8994b949f8b90da11a3e66b is 50, key is test_row_0/B:col10/1731735601469/Put/seqid=0 2024-11-16T05:40:03,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741955_1131 (size=12301) 2024-11-16T05:40:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T05:40:04,123 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/f38d1db4f8994b949f8b90da11a3e66b 2024-11-16T05:40:04,133 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/c6410326abee4eef9095dcc26fc6fa13 is 50, key is test_row_0/C:col10/1731735601469/Put/seqid=0 2024-11-16T05:40:04,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741956_1132 (size=12301) 2024-11-16T05:40:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T05:40:04,539 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/c6410326abee4eef9095dcc26fc6fa13 2024-11-16T05:40:04,545 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/A/e81200c052314a8f9cd708142e689e8f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e81200c052314a8f9cd708142e689e8f 2024-11-16T05:40:04,550 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e81200c052314a8f9cd708142e689e8f, entries=150, sequenceid=536, filesize=12.0 K 2024-11-16T05:40:04,551 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/B/f38d1db4f8994b949f8b90da11a3e66b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/f38d1db4f8994b949f8b90da11a3e66b 2024-11-16T05:40:04,556 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/f38d1db4f8994b949f8b90da11a3e66b, entries=150, sequenceid=536, filesize=12.0 K 2024-11-16T05:40:04,557 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/.tmp/C/c6410326abee4eef9095dcc26fc6fa13 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c6410326abee4eef9095dcc26fc6fa13 2024-11-16T05:40:04,561 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c6410326abee4eef9095dcc26fc6fa13, entries=150, sequenceid=536, filesize=12.0 K 2024-11-16T05:40:04,562 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3992ee2db1edacc4ba76f5d2411f31eb in 1271ms, sequenceid=536, compaction requested=true 2024-11-16T05:40:04,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ad9914c52f004ee88c414c128a537730, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5fa1c4e2801a4ab8922c6096da40ecd6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9e5fd900c240470aa946c514af6a5bea, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5579d2c602084aa29dbd5106687ac5dd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/7a41c58b6d3247bf8c961dd58f757e38, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/d4f471ab3b224cdaa5ef7ebba2d1d9d0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/11cc37003f4c4a1fbeacf6ac1de047be, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f0cf24ab02fc4c0288ee2b811440bdc8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/96974b5534c541a294107894c23efbc2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5ae92d0ab2d54a108e5bbf877bc3b636, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e34a3a50f2e145018ea80179d9bbc615, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f421913d38344673bc6af2b180cb2ed6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9bc048fba9ae4c5eac2b89db6e75b0d6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a8cd760548b54b0c959660bcd884bb5f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b7d2b92a3f964a01afcf039ca33dd114, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f292166f9b184bceb9cda79c0596a89c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/1513eab2a2c04770a63a52230190a441, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a45fe28fc8284d8aa27e37a3193d889c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/892eb8574dbb440ba5b6f964caaec63a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4806287c044f49d49aa56d05bfd3492d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/6b7d6c09334a42d4a53ca441c055e4b4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9183f6ca411d46f48574b2e72f7157c5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/722a56f90c154cf895c32f119c0a1f1b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/150e05b6e9da4752be5c64c6cbe2b53b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/2ab2142236734040a2ff6f8b60127129, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0c59da645cd408195bb3bb5f8f17df8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/61c5eba1cb4e4a43ad0a094556aadb46, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b5d58e63463b4704b62deb5d0d7b7821, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/0f0eae2a31bb49efab4a1c73e2de4750, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a103b66b57ca409ab14bf411c38da3f0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/c156f9303c014b5280d133971fc5b6d6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ab628ea0c995412e993af2358b1d5ad0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0d9f1224c91458b8693f7764f1955fd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/831223de128040a2925eb4068b1cb9d1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/272fd54da487415f9c7493277dcf2310, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4d3da80cd98f475a85288c082b8f18b4] to archive 2024-11-16T05:40:04,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:40:04,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ad9914c52f004ee88c414c128a537730 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ad9914c52f004ee88c414c128a537730 2024-11-16T05:40:04,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5fa1c4e2801a4ab8922c6096da40ecd6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5fa1c4e2801a4ab8922c6096da40ecd6 2024-11-16T05:40:04,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9e5fd900c240470aa946c514af6a5bea to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9e5fd900c240470aa946c514af6a5bea 2024-11-16T05:40:04,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5579d2c602084aa29dbd5106687ac5dd to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5579d2c602084aa29dbd5106687ac5dd 2024-11-16T05:40:04,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/7a41c58b6d3247bf8c961dd58f757e38 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/7a41c58b6d3247bf8c961dd58f757e38 2024-11-16T05:40:04,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/d4f471ab3b224cdaa5ef7ebba2d1d9d0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/d4f471ab3b224cdaa5ef7ebba2d1d9d0 2024-11-16T05:40:04,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/11cc37003f4c4a1fbeacf6ac1de047be to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/11cc37003f4c4a1fbeacf6ac1de047be 2024-11-16T05:40:04,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f0cf24ab02fc4c0288ee2b811440bdc8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f0cf24ab02fc4c0288ee2b811440bdc8 2024-11-16T05:40:04,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/96974b5534c541a294107894c23efbc2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/96974b5534c541a294107894c23efbc2 2024-11-16T05:40:04,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5ae92d0ab2d54a108e5bbf877bc3b636 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/5ae92d0ab2d54a108e5bbf877bc3b636 2024-11-16T05:40:04,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e34a3a50f2e145018ea80179d9bbc615 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e34a3a50f2e145018ea80179d9bbc615 2024-11-16T05:40:04,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f421913d38344673bc6af2b180cb2ed6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f421913d38344673bc6af2b180cb2ed6 2024-11-16T05:40:04,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9bc048fba9ae4c5eac2b89db6e75b0d6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9bc048fba9ae4c5eac2b89db6e75b0d6 2024-11-16T05:40:04,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a8cd760548b54b0c959660bcd884bb5f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a8cd760548b54b0c959660bcd884bb5f 2024-11-16T05:40:04,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b7d2b92a3f964a01afcf039ca33dd114 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b7d2b92a3f964a01afcf039ca33dd114 2024-11-16T05:40:04,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f292166f9b184bceb9cda79c0596a89c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/f292166f9b184bceb9cda79c0596a89c 2024-11-16T05:40:04,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/1513eab2a2c04770a63a52230190a441 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/1513eab2a2c04770a63a52230190a441 2024-11-16T05:40:04,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a45fe28fc8284d8aa27e37a3193d889c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a45fe28fc8284d8aa27e37a3193d889c 2024-11-16T05:40:04,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/892eb8574dbb440ba5b6f964caaec63a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/892eb8574dbb440ba5b6f964caaec63a 2024-11-16T05:40:04,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4806287c044f49d49aa56d05bfd3492d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4806287c044f49d49aa56d05bfd3492d 2024-11-16T05:40:04,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/6b7d6c09334a42d4a53ca441c055e4b4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/6b7d6c09334a42d4a53ca441c055e4b4 2024-11-16T05:40:04,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9183f6ca411d46f48574b2e72f7157c5 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/9183f6ca411d46f48574b2e72f7157c5 2024-11-16T05:40:04,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/722a56f90c154cf895c32f119c0a1f1b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/722a56f90c154cf895c32f119c0a1f1b 2024-11-16T05:40:04,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/150e05b6e9da4752be5c64c6cbe2b53b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/150e05b6e9da4752be5c64c6cbe2b53b 2024-11-16T05:40:04,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/2ab2142236734040a2ff6f8b60127129 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/2ab2142236734040a2ff6f8b60127129 2024-11-16T05:40:04,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0c59da645cd408195bb3bb5f8f17df8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0c59da645cd408195bb3bb5f8f17df8 2024-11-16T05:40:04,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/61c5eba1cb4e4a43ad0a094556aadb46 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/61c5eba1cb4e4a43ad0a094556aadb46 2024-11-16T05:40:04,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b5d58e63463b4704b62deb5d0d7b7821 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b5d58e63463b4704b62deb5d0d7b7821 2024-11-16T05:40:04,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/0f0eae2a31bb49efab4a1c73e2de4750 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/0f0eae2a31bb49efab4a1c73e2de4750 2024-11-16T05:40:04,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a103b66b57ca409ab14bf411c38da3f0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/a103b66b57ca409ab14bf411c38da3f0 2024-11-16T05:40:04,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/c156f9303c014b5280d133971fc5b6d6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/c156f9303c014b5280d133971fc5b6d6 2024-11-16T05:40:04,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ab628ea0c995412e993af2358b1d5ad0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/ab628ea0c995412e993af2358b1d5ad0 2024-11-16T05:40:04,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0d9f1224c91458b8693f7764f1955fd to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/b0d9f1224c91458b8693f7764f1955fd 2024-11-16T05:40:04,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/831223de128040a2925eb4068b1cb9d1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/831223de128040a2925eb4068b1cb9d1 2024-11-16T05:40:04,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/272fd54da487415f9c7493277dcf2310 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/272fd54da487415f9c7493277dcf2310 2024-11-16T05:40:04,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4d3da80cd98f475a85288c082b8f18b4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/4d3da80cd98f475a85288c082b8f18b4 2024-11-16T05:40:04,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/dc69df6d0d4044a0ac5e93a6780f864d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3c9c0c32254447f28efd513ba624827c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/e789e25f26284daebc4a4c7b3e66bbc0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/71b079b4fdcc4277ae2058bef03a3cf7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/87aa1e35ec4b4f60ad69be9c6ca78ae0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/02e070c21ec84261844281b581461619, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/d53ae02c4a924070a37390bcb55aade0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/9d03ff69455c433d9816f22cd1194de2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/1785f360c27a4291936f11902a8b0ae9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6c6eb72272c7431da062381bd37f6b37, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/18a4046c07214170a35dfee847c58a68, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/e3cdcc5a3cf0484db93e3aa238e8fe39, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ad56ed6f566947d4ae1eaafbe76818e4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/aa219ec622944c598b152f056e30a07d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ef3168b17d2c44a3924cbee21f2747b5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/38ba4342703f44e19fd9ce5d3b75b75c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2d92f19dba964766a7b453468d6f1a3e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/566962c3a4544487932d7a19a35ef8a3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4bbb15cfc10b4d2ab5930956fa3ae3c4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/27e8e697b873474aae3342aca7ba77b9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/449dd95578154edda67c11a449fea073, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/027468104b9d47bc868667c53ba0f127, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/b3efd0ca0aa5450db536f2df74331ae2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/c83346d322ee4a7cb1b3e935bc86b1c6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/49aa8d3b3de24b84ac34ad3cbba65c62, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/08ebaa214bde43d2a511396e42c292e5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3ca0e8a091c74801b1b80d90da791614, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4ad6bf36ddd2496ea788bb437ebce0ce, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6639610573e34ed0a436779779e9ba5e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2e285cee25a04ed9870b38e08c9429a0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/51c871147452410da79f9d89ef9c702b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/695802fa19f74553b7d9eb3cc0480fe8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2bdd332649674b0ea102be6ccd87f850, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/34eff7e9e6ab45c3b6aedc5e2c32913d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/a3c35aa43a41445c8a86a0ca6cb5bf29, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ea1e4f60fe964905b47e4fbf7dc3b028] to archive 2024-11-16T05:40:04,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:40:04,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/dc69df6d0d4044a0ac5e93a6780f864d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/dc69df6d0d4044a0ac5e93a6780f864d 2024-11-16T05:40:04,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3c9c0c32254447f28efd513ba624827c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3c9c0c32254447f28efd513ba624827c 2024-11-16T05:40:04,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/e789e25f26284daebc4a4c7b3e66bbc0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/e789e25f26284daebc4a4c7b3e66bbc0 2024-11-16T05:40:04,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/71b079b4fdcc4277ae2058bef03a3cf7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/71b079b4fdcc4277ae2058bef03a3cf7 2024-11-16T05:40:04,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/87aa1e35ec4b4f60ad69be9c6ca78ae0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/87aa1e35ec4b4f60ad69be9c6ca78ae0 2024-11-16T05:40:04,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/02e070c21ec84261844281b581461619 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/02e070c21ec84261844281b581461619 2024-11-16T05:40:04,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/d53ae02c4a924070a37390bcb55aade0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/d53ae02c4a924070a37390bcb55aade0 2024-11-16T05:40:04,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/9d03ff69455c433d9816f22cd1194de2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/9d03ff69455c433d9816f22cd1194de2 2024-11-16T05:40:04,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/1785f360c27a4291936f11902a8b0ae9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/1785f360c27a4291936f11902a8b0ae9 2024-11-16T05:40:04,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6c6eb72272c7431da062381bd37f6b37 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6c6eb72272c7431da062381bd37f6b37 2024-11-16T05:40:04,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/18a4046c07214170a35dfee847c58a68 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/18a4046c07214170a35dfee847c58a68 2024-11-16T05:40:04,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/e3cdcc5a3cf0484db93e3aa238e8fe39 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/e3cdcc5a3cf0484db93e3aa238e8fe39 2024-11-16T05:40:04,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ad56ed6f566947d4ae1eaafbe76818e4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ad56ed6f566947d4ae1eaafbe76818e4 2024-11-16T05:40:04,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/aa219ec622944c598b152f056e30a07d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/aa219ec622944c598b152f056e30a07d 2024-11-16T05:40:04,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ef3168b17d2c44a3924cbee21f2747b5 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ef3168b17d2c44a3924cbee21f2747b5 2024-11-16T05:40:04,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/38ba4342703f44e19fd9ce5d3b75b75c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/38ba4342703f44e19fd9ce5d3b75b75c 2024-11-16T05:40:04,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2d92f19dba964766a7b453468d6f1a3e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2d92f19dba964766a7b453468d6f1a3e 2024-11-16T05:40:04,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/566962c3a4544487932d7a19a35ef8a3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/566962c3a4544487932d7a19a35ef8a3 2024-11-16T05:40:04,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4bbb15cfc10b4d2ab5930956fa3ae3c4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4bbb15cfc10b4d2ab5930956fa3ae3c4 2024-11-16T05:40:04,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/27e8e697b873474aae3342aca7ba77b9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/27e8e697b873474aae3342aca7ba77b9 2024-11-16T05:40:04,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/449dd95578154edda67c11a449fea073 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/449dd95578154edda67c11a449fea073 2024-11-16T05:40:04,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/027468104b9d47bc868667c53ba0f127 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/027468104b9d47bc868667c53ba0f127 2024-11-16T05:40:04,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/b3efd0ca0aa5450db536f2df74331ae2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/b3efd0ca0aa5450db536f2df74331ae2 2024-11-16T05:40:04,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/c83346d322ee4a7cb1b3e935bc86b1c6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/c83346d322ee4a7cb1b3e935bc86b1c6 2024-11-16T05:40:04,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/49aa8d3b3de24b84ac34ad3cbba65c62 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/49aa8d3b3de24b84ac34ad3cbba65c62 2024-11-16T05:40:04,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/08ebaa214bde43d2a511396e42c292e5 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/08ebaa214bde43d2a511396e42c292e5 2024-11-16T05:40:04,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3ca0e8a091c74801b1b80d90da791614 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/3ca0e8a091c74801b1b80d90da791614 2024-11-16T05:40:04,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4ad6bf36ddd2496ea788bb437ebce0ce to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/4ad6bf36ddd2496ea788bb437ebce0ce 2024-11-16T05:40:04,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6639610573e34ed0a436779779e9ba5e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/6639610573e34ed0a436779779e9ba5e 2024-11-16T05:40:04,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2e285cee25a04ed9870b38e08c9429a0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2e285cee25a04ed9870b38e08c9429a0 2024-11-16T05:40:04,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/51c871147452410da79f9d89ef9c702b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/51c871147452410da79f9d89ef9c702b 2024-11-16T05:40:04,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/695802fa19f74553b7d9eb3cc0480fe8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/695802fa19f74553b7d9eb3cc0480fe8 2024-11-16T05:40:04,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2bdd332649674b0ea102be6ccd87f850 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/2bdd332649674b0ea102be6ccd87f850 2024-11-16T05:40:04,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/34eff7e9e6ab45c3b6aedc5e2c32913d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/34eff7e9e6ab45c3b6aedc5e2c32913d 2024-11-16T05:40:04,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/a3c35aa43a41445c8a86a0ca6cb5bf29 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/a3c35aa43a41445c8a86a0ca6cb5bf29 2024-11-16T05:40:04,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ea1e4f60fe964905b47e4fbf7dc3b028 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/ea1e4f60fe964905b47e4fbf7dc3b028 2024-11-16T05:40:04,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/cb0e73e78f9142639be4dfe235692598, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/3701284dfa2047bcacf46b258224328f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f5cbbb582850407b913f2dcd33396325, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f57b16f04269471cad536c8b6cca2867, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5917cf418ec049dc84f57bb0ceab0b04, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7e0f338d82004a51951d221f3a1a9b7c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/321c1a2b40fa4437b36f19f5a1194940, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c11378ad0279453287407de74325fb9a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/e732865aa73343eda8e66a942d3d73a0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/94347bdd1ea44220b698229c0359c192, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ff4deb02f1b9422a8957dcbd4bd64f1f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/e66f487470f548a183601211e135d780, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7c5aa4ffa00b47d786d2e21c587051fa, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/a34a4730577149af8bf7ca8bebb56806, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/8db3525a7b8943d9a1cd978657f5287e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/978b7d4b012d45f4a7eec1e488b571c6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ac11e1f97a744cc8a22820e479fa2f3e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/2096c60c4da04725b704d2e812b16a63, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4e6d7bb7da944ffa9b5a69c37df3fba3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/6b50d90db79f4c6a8189b0febee877c4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5a15d4ec0a534b9a89c9be1a5b473729, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81912b1b9e6d46c08654c0b183a92001, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/68b7c640c1374085b2f91343d261dfab, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/d29fe685329e44018ec6dc49563ef4c9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/91875ec6d5df47558f97eebce9826e0d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5e60c5d7adc642a78c96a973aab55268, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81efb738e3f14690a5bd6f4a7848a540, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7a538c9b9620477ba3e3886107dd52e2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/feb3365cef7640b1b3b1572bd5aeb742, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ebc35b139d7b495e9813d5de0b26bab7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/9128dd03eea64d15be15b7a3b23c2007, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/dcb92e8bba334cc58c209ede187fa17e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4b836914d55e41b7ae5c3167dcd3a4b5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/92f6e740cf8d4e4e8baaa5bc34bfc852, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/16ba291bdd594319a235fff9f96fa31c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/60b85942cb144b9da841bccc3b76f04f] to archive 2024-11-16T05:40:04,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:40:04,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/cb0e73e78f9142639be4dfe235692598 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/cb0e73e78f9142639be4dfe235692598 2024-11-16T05:40:04,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/3701284dfa2047bcacf46b258224328f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/3701284dfa2047bcacf46b258224328f 2024-11-16T05:40:04,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f5cbbb582850407b913f2dcd33396325 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f5cbbb582850407b913f2dcd33396325 2024-11-16T05:40:04,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f57b16f04269471cad536c8b6cca2867 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/f57b16f04269471cad536c8b6cca2867 2024-11-16T05:40:04,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5917cf418ec049dc84f57bb0ceab0b04 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5917cf418ec049dc84f57bb0ceab0b04 2024-11-16T05:40:04,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7e0f338d82004a51951d221f3a1a9b7c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7e0f338d82004a51951d221f3a1a9b7c 2024-11-16T05:40:04,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/321c1a2b40fa4437b36f19f5a1194940 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/321c1a2b40fa4437b36f19f5a1194940 2024-11-16T05:40:04,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c11378ad0279453287407de74325fb9a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c11378ad0279453287407de74325fb9a 2024-11-16T05:40:04,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/e732865aa73343eda8e66a942d3d73a0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/e732865aa73343eda8e66a942d3d73a0 2024-11-16T05:40:04,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/94347bdd1ea44220b698229c0359c192 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/94347bdd1ea44220b698229c0359c192 2024-11-16T05:40:04,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ff4deb02f1b9422a8957dcbd4bd64f1f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ff4deb02f1b9422a8957dcbd4bd64f1f 2024-11-16T05:40:04,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/e66f487470f548a183601211e135d780 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/e66f487470f548a183601211e135d780 2024-11-16T05:40:04,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7c5aa4ffa00b47d786d2e21c587051fa to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7c5aa4ffa00b47d786d2e21c587051fa 2024-11-16T05:40:04,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/a34a4730577149af8bf7ca8bebb56806 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/a34a4730577149af8bf7ca8bebb56806 2024-11-16T05:40:04,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/8db3525a7b8943d9a1cd978657f5287e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/8db3525a7b8943d9a1cd978657f5287e 2024-11-16T05:40:04,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/978b7d4b012d45f4a7eec1e488b571c6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/978b7d4b012d45f4a7eec1e488b571c6 2024-11-16T05:40:04,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ac11e1f97a744cc8a22820e479fa2f3e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ac11e1f97a744cc8a22820e479fa2f3e 2024-11-16T05:40:04,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/2096c60c4da04725b704d2e812b16a63 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/2096c60c4da04725b704d2e812b16a63 2024-11-16T05:40:04,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4e6d7bb7da944ffa9b5a69c37df3fba3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4e6d7bb7da944ffa9b5a69c37df3fba3 2024-11-16T05:40:04,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/6b50d90db79f4c6a8189b0febee877c4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/6b50d90db79f4c6a8189b0febee877c4 2024-11-16T05:40:04,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5a15d4ec0a534b9a89c9be1a5b473729 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5a15d4ec0a534b9a89c9be1a5b473729 2024-11-16T05:40:04,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81912b1b9e6d46c08654c0b183a92001 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81912b1b9e6d46c08654c0b183a92001 2024-11-16T05:40:04,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/68b7c640c1374085b2f91343d261dfab to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/68b7c640c1374085b2f91343d261dfab 2024-11-16T05:40:04,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/d29fe685329e44018ec6dc49563ef4c9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/d29fe685329e44018ec6dc49563ef4c9 2024-11-16T05:40:04,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/91875ec6d5df47558f97eebce9826e0d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/91875ec6d5df47558f97eebce9826e0d 2024-11-16T05:40:04,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5e60c5d7adc642a78c96a973aab55268 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/5e60c5d7adc642a78c96a973aab55268 2024-11-16T05:40:04,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81efb738e3f14690a5bd6f4a7848a540 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/81efb738e3f14690a5bd6f4a7848a540 2024-11-16T05:40:04,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7a538c9b9620477ba3e3886107dd52e2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/7a538c9b9620477ba3e3886107dd52e2 2024-11-16T05:40:04,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/feb3365cef7640b1b3b1572bd5aeb742 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/feb3365cef7640b1b3b1572bd5aeb742 2024-11-16T05:40:04,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ebc35b139d7b495e9813d5de0b26bab7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/ebc35b139d7b495e9813d5de0b26bab7 2024-11-16T05:40:04,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/9128dd03eea64d15be15b7a3b23c2007 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/9128dd03eea64d15be15b7a3b23c2007 2024-11-16T05:40:04,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/dcb92e8bba334cc58c209ede187fa17e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/dcb92e8bba334cc58c209ede187fa17e 2024-11-16T05:40:04,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4b836914d55e41b7ae5c3167dcd3a4b5 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/4b836914d55e41b7ae5c3167dcd3a4b5 2024-11-16T05:40:04,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/92f6e740cf8d4e4e8baaa5bc34bfc852 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/92f6e740cf8d4e4e8baaa5bc34bfc852 2024-11-16T05:40:04,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/16ba291bdd594319a235fff9f96fa31c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/16ba291bdd594319a235fff9f96fa31c 2024-11-16T05:40:04,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/60b85942cb144b9da841bccc3b76f04f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/60b85942cb144b9da841bccc3b76f04f 2024-11-16T05:40:04,741 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/recovered.edits/539.seqid, newMaxSeqId=539, maxSeqId=1 2024-11-16T05:40:04,742 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. 2024-11-16T05:40:04,742 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 3992ee2db1edacc4ba76f5d2411f31eb: Waiting for close lock at 1731735603291Running coprocessor pre-close hooks at 1731735603291Disabling compacts and flushes for region at 1731735603291Disabling writes for close at 1731735603291Obtaining lock to block concurrent updates at 1731735603291Preparing flush snapshotting stores in 3992ee2db1edacc4ba76f5d2411f31eb at 1731735603291Finished memstore snapshotting TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731735603292 (+1 ms)Flushing stores of TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb. at 1731735603292Flushing 3992ee2db1edacc4ba76f5d2411f31eb/A: creating writer at 1731735603293 (+1 ms)Flushing 3992ee2db1edacc4ba76f5d2411f31eb/A: appending metadata at 1731735603297 (+4 ms)Flushing 3992ee2db1edacc4ba76f5d2411f31eb/A: closing flushed file at 1731735603297Flushing 3992ee2db1edacc4ba76f5d2411f31eb/B: creating writer at 1731735603710 (+413 ms)Flushing 3992ee2db1edacc4ba76f5d2411f31eb/B: appending metadata at 1731735603717 (+7 ms)Flushing 3992ee2db1edacc4ba76f5d2411f31eb/B: closing flushed file at 1731735603717Flushing 3992ee2db1edacc4ba76f5d2411f31eb/C: creating writer at 1731735604129 (+412 ms)Flushing 3992ee2db1edacc4ba76f5d2411f31eb/C: appending metadata at 1731735604132 (+3 ms)Flushing 3992ee2db1edacc4ba76f5d2411f31eb/C: closing flushed file at 1731735604133 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74bedefb: reopening flushed file at 1731735604544 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c5fb673: reopening flushed file at 1731735604550 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20a6b73c: reopening flushed file at 1731735604556 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3992ee2db1edacc4ba76f5d2411f31eb in 1271ms, sequenceid=536, compaction requested=true at 1731735604562 (+6 ms)Writing region close event to WAL at 1731735604738 (+176 ms)Running coprocessor post-close hooks at 1731735604742 (+4 ms)Closed at 1731735604742 2024-11-16T05:40:04,744 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:40:04,745 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3992ee2db1edacc4ba76f5d2411f31eb, regionState=CLOSED 2024-11-16T05:40:04,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:40:04,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-16T05:40:04,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 3992ee2db1edacc4ba76f5d2411f31eb, server=3456ee6a3164,35555,1731735575219 in 1.6160 sec 2024-11-16T05:40:04,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-16T05:40:04,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3992ee2db1edacc4ba76f5d2411f31eb, UNASSIGN in 1.6270 sec 2024-11-16T05:40:04,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-16T05:40:04,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6370 sec 2024-11-16T05:40:04,758 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735604758"}]},"ts":"1731735604758"} 2024-11-16T05:40:04,760 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:40:04,760 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:40:04,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6520 sec 2024-11-16T05:40:04,949 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:40:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T05:40:05,259 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:05,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:40:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:05,273 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T05:40:05,275 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:05,281 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:40:05,286 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/recovered.edits] 2024-11-16T05:40:05,289 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/1462f25ea75340f293bb49826cea63e9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/1462f25ea75340f293bb49826cea63e9 2024-11-16T05:40:05,291 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e2ff8f6efc9744879a3a0347e7edd1c4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e2ff8f6efc9744879a3a0347e7edd1c4 2024-11-16T05:40:05,292 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e81200c052314a8f9cd708142e689e8f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e81200c052314a8f9cd708142e689e8f 2024-11-16T05:40:05,294 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e93d3ac8fb724573959449c442bd1045 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/A/e93d3ac8fb724573959449c442bd1045 2024-11-16T05:40:05,296 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/1caf9e9d25fc4140986d8a1129f0d9e9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/1caf9e9d25fc4140986d8a1129f0d9e9 2024-11-16T05:40:05,297 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/89d4d3dabb404526a884aa3263d6010c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/89d4d3dabb404526a884aa3263d6010c 2024-11-16T05:40:05,298 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/f38d1db4f8994b949f8b90da11a3e66b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/f38d1db4f8994b949f8b90da11a3e66b 2024-11-16T05:40:05,300 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/f75bee6dd30b45a19254545113bc346f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/B/f75bee6dd30b45a19254545113bc346f 2024-11-16T05:40:05,302 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/b01906ba322f4ce793c365dcb3d7dc26 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/b01906ba322f4ce793c365dcb3d7dc26 2024-11-16T05:40:05,303 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c6410326abee4eef9095dcc26fc6fa13 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c6410326abee4eef9095dcc26fc6fa13 2024-11-16T05:40:05,304 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c93959c049094791ab94dfa14d09e939 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/c93959c049094791ab94dfa14d09e939 2024-11-16T05:40:05,306 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/d70d6998c57f46998c5602dea01cab5c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/C/d70d6998c57f46998c5602dea01cab5c 2024-11-16T05:40:05,308 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/recovered.edits/539.seqid to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb/recovered.edits/539.seqid 2024-11-16T05:40:05,309 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/3992ee2db1edacc4ba76f5d2411f31eb 2024-11-16T05:40:05,309 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:40:05,313 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:05,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-16T05:40:05,324 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:40:05,353 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:40:05,355 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:05,355 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:40:05,355 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731735605355"}]},"ts":"9223372036854775807"} 2024-11-16T05:40:05,360 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:40:05,360 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3992ee2db1edacc4ba76f5d2411f31eb, NAME => 'TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:40:05,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:40:05,360 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731735605360"}]},"ts":"9223372036854775807"} 2024-11-16T05:40:05,364 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:40:05,365 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:05,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 99 msec 2024-11-16T05:40:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T05:40:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:40:05,379 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:05,390 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1185263931_22 at /127.0.0.1:35956 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1836729063_22 at /127.0.0.1:43986 [Waiting for operation #371] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1185263931_22 at /127.0.0.1:47358 [Waiting for operation #356] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;3456ee6a3164:35555-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=507 (was 366) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3559 (was 4329) 2024-11-16T05:40:05,399 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=507, ProcessCount=11, AvailableMemoryMB=3559 2024-11-16T05:40:05,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:40:05,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:40:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:05,404 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:40:05,404 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:05,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-16T05:40:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-16T05:40:05,405 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:40:05,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741957_1133 (size=959) 2024-11-16T05:40:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-16T05:40:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-16T05:40:05,817 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d 2024-11-16T05:40:05,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741958_1134 (size=53) 2024-11-16T05:40:05,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:40:05,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 82b1d437a2b8fce4a136c21be6257221, disabling compactions & flushes 2024-11-16T05:40:05,828 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:05,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:05,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. after waiting 0 ms 2024-11-16T05:40:05,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:05,828 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:05,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 82b1d437a2b8fce4a136c21be6257221: Waiting for close lock at 1731735605827Disabling compacts and flushes for region at 1731735605827Disabling writes for close at 1731735605828 (+1 ms)Writing region close event to WAL at 1731735605828Closed at 1731735605828 2024-11-16T05:40:05,829 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:40:05,830 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731735605829"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731735605829"}]},"ts":"1731735605829"} 2024-11-16T05:40:05,832 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:40:05,834 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:40:05,834 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735605834"}]},"ts":"1731735605834"} 2024-11-16T05:40:05,837 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:40:05,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, ASSIGN}] 2024-11-16T05:40:05,839 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, ASSIGN 2024-11-16T05:40:05,840 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, ASSIGN; state=OFFLINE, location=3456ee6a3164,35555,1731735575219; forceNewPlan=false, retain=false 2024-11-16T05:40:05,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=82b1d437a2b8fce4a136c21be6257221, regionState=OPENING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:05,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, ASSIGN because future has completed 2024-11-16T05:40:05,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:40:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-16T05:40:06,150 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:06,150 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:40:06,150 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,150 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:40:06,151 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,151 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,153 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,155 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b1d437a2b8fce4a136c21be6257221 columnFamilyName A 2024-11-16T05:40:06,155 DEBUG [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:06,156 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(327): Store=82b1d437a2b8fce4a136c21be6257221/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:40:06,156 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,157 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b1d437a2b8fce4a136c21be6257221 columnFamilyName B 2024-11-16T05:40:06,157 DEBUG [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:06,158 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(327): Store=82b1d437a2b8fce4a136c21be6257221/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:40:06,158 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,159 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b1d437a2b8fce4a136c21be6257221 columnFamilyName C 2024-11-16T05:40:06,159 DEBUG [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:06,159 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(327): Store=82b1d437a2b8fce4a136c21be6257221/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:40:06,159 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,160 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,160 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,161 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,161 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,162 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:40:06,163 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,165 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:40:06,165 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 82b1d437a2b8fce4a136c21be6257221; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62062890, jitterRate=-0.07519087195396423}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:40:06,165 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:06,166 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 82b1d437a2b8fce4a136c21be6257221: Running coprocessor pre-open hook at 1731735606151Writing region info on filesystem at 1731735606151Initializing all the Stores at 1731735606152 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735606152Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735606152Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735606152Cleaning up temporary data from old regions at 1731735606161 (+9 ms)Running coprocessor post-open hooks at 1731735606165 (+4 ms)Region opened successfully at 1731735606166 (+1 ms) 2024-11-16T05:40:06,167 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., pid=36, masterSystemTime=1731735606145 2024-11-16T05:40:06,169 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:06,169 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:06,169 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=82b1d437a2b8fce4a136c21be6257221, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:06,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:40:06,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-16T05:40:06,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 in 179 msec 2024-11-16T05:40:06,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-16T05:40:06,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, ASSIGN in 337 msec 2024-11-16T05:40:06,176 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:40:06,177 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735606176"}]},"ts":"1731735606176"} 2024-11-16T05:40:06,178 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:40:06,179 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:40:06,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 778 msec 2024-11-16T05:40:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-16T05:40:06,539 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:06,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:06,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:06,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:06,544 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f9a034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:06,547 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@810766a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:06,548 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:06,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:06,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:06,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:06,553 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:40:06,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,39409,1731735574536 2024-11-16T05:40:06,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79c1019a 2024-11-16T05:40:06,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:40:06,556 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:40:06,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:40:06,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:40:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:06,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741959_1135 (size=995) 2024-11-16T05:40:06,983 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T05:40:06,983 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T05:40:06,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:40:06,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, REOPEN/MOVE}] 2024-11-16T05:40:06,996 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, REOPEN/MOVE 2024-11-16T05:40:06,997 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=82b1d437a2b8fce4a136c21be6257221, regionState=CLOSING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:06,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, REOPEN/MOVE because future has completed 2024-11-16T05:40:06,999 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:40:06,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:40:07,152 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,153 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:40:07,153 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 82b1d437a2b8fce4a136c21be6257221, disabling compactions & flushes 2024-11-16T05:40:07,153 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,153 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,153 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. after waiting 0 ms 2024-11-16T05:40:07,153 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,156 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:40:07,157 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,157 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 82b1d437a2b8fce4a136c21be6257221: Waiting for close lock at 1731735607153Running coprocessor pre-close hooks at 1731735607153Disabling compacts and flushes for region at 1731735607153Disabling writes for close at 1731735607153Writing region close event to WAL at 1731735607153Running coprocessor post-close hooks at 1731735607157 (+4 ms)Closed at 1731735607157 2024-11-16T05:40:07,157 WARN [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 82b1d437a2b8fce4a136c21be6257221 to self. 2024-11-16T05:40:07,159 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,159 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=82b1d437a2b8fce4a136c21be6257221, regionState=CLOSED 2024-11-16T05:40:07,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:40:07,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-16T05:40:07,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 in 163 msec 2024-11-16T05:40:07,164 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, REOPEN/MOVE; state=CLOSED, location=3456ee6a3164,35555,1731735575219; forceNewPlan=false, retain=true 2024-11-16T05:40:07,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=82b1d437a2b8fce4a136c21be6257221, regionState=OPENING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, REOPEN/MOVE because future has completed 2024-11-16T05:40:07,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:40:07,477 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,477 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:40:07,478 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,478 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:40:07,478 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,478 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,483 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,490 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b1d437a2b8fce4a136c21be6257221 columnFamilyName A 2024-11-16T05:40:07,492 DEBUG [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:07,493 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(327): Store=82b1d437a2b8fce4a136c21be6257221/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:40:07,493 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,494 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b1d437a2b8fce4a136c21be6257221 columnFamilyName B 2024-11-16T05:40:07,494 DEBUG [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:07,495 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(327): Store=82b1d437a2b8fce4a136c21be6257221/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:40:07,495 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,495 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b1d437a2b8fce4a136c21be6257221 columnFamilyName C 2024-11-16T05:40:07,495 DEBUG [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:07,496 INFO [StoreOpener-82b1d437a2b8fce4a136c21be6257221-1 {}] regionserver.HStore(327): Store=82b1d437a2b8fce4a136c21be6257221/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:40:07,496 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,497 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,498 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,499 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,499 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,500 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:40:07,501 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,502 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 82b1d437a2b8fce4a136c21be6257221; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67072496, jitterRate=-5.419254302978516E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:40:07,502 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,503 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 82b1d437a2b8fce4a136c21be6257221: Running coprocessor pre-open hook at 1731735607478Writing region info on filesystem at 1731735607478Initializing all the Stores at 1731735607479 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735607479Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735607479Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735607479Cleaning up temporary data from old regions at 1731735607499 (+20 ms)Running coprocessor post-open hooks at 1731735607502 (+3 ms)Region opened successfully at 1731735607503 (+1 ms) 2024-11-16T05:40:07,504 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., pid=41, masterSystemTime=1731735607472 2024-11-16T05:40:07,506 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,506 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,506 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=82b1d437a2b8fce4a136c21be6257221, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:40:07,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-16T05:40:07,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 in 190 msec 2024-11-16T05:40:07,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-16T05:40:07,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, REOPEN/MOVE in 516 msec 2024-11-16T05:40:07,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-16T05:40:07,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 527 msec 2024-11-16T05:40:07,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 957 msec 2024-11-16T05:40:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-16T05:40:07,525 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:07,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:07,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:07,527 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee70f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:07,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:07,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:07,529 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4474113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:07,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:07,532 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:07,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad5e6af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:07,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:07,534 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f251491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:07,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:07,537 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:07,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:07,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@278f3cdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:07,539 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@112cdd11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:07,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:07,542 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c90612, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:07,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:07,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:07,544 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:07,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d70fc20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:07,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:07,549 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17815ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:07,551 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70386680, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:07,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:07,554 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c02472a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:07,555 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66040825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:07,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:07,558 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:07,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:07,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:07,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f30a7f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:07,560 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eea8ecf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:07,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:07,562 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@118dd7e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:07,564 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:07,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5625b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:07,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:07,567 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37937e39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:07,569 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d64e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:07,571 DEBUG [Thread-679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:07,571 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:07,571 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:07,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:07,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:07,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:07,577 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:07,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:40:07,577 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:07,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:40:07,578 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:07,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:07,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:40:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-16T05:40:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:07,580 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:40:07,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:07,581 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:07,581 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:07,581 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:07,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:07,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:07,582 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:07,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:07,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:07,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:07,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:07,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:07,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:07,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:07,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:40:07,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:07,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:40:07,587 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:07,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:07,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:40:07,589 DEBUG [Thread-686 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:07,589 DEBUG [Thread-686 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:40:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:40:07,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:07,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:40:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,600 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:07,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T05:40:07,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163bf7b4c0b96a4f869627bfddb339fcd5_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735607592/Put/seqid=0 2024-11-16T05:40:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:40:07,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735667673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735667674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735667676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:07,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735667680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:07,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735667689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:07,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741960_1136 (size=12154) 2024-11-16T05:40:07,706 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:07,712 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163bf7b4c0b96a4f869627bfddb339fcd5_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163bf7b4c0b96a4f869627bfddb339fcd5_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:07,722 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/15dc7b60cfea4b159c98f557dcd58277, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:07,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/15dc7b60cfea4b159c98f557dcd58277 is 175, key is test_row_0/A:col10/1731735607592/Put/seqid=0 2024-11-16T05:40:07,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:40:07,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:07,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:07,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:07,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741961_1137 (size=30955) 2024-11-16T05:40:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735667798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735667798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735667798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:07,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735667798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:07,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735667805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:07,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:40:07,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:07,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:07,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:40:08,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735668009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735668019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735668019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735668019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735668019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:40:08,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:08,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:08,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:08,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,158 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/15dc7b60cfea4b159c98f557dcd58277 2024-11-16T05:40:08,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:40:08,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:08,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:08,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:08,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/605797a240684449a029c27ac5354808 is 50, key is test_row_0/B:col10/1731735607592/Put/seqid=0 2024-11-16T05:40:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:40:08,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741962_1138 (size=12001) 2024-11-16T05:40:08,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/605797a240684449a029c27ac5354808 2024-11-16T05:40:08,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/d3a1c0fbfc354e9094b5efa972ece3e0 is 50, key is test_row_0/C:col10/1731735607592/Put/seqid=0 2024-11-16T05:40:08,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741963_1139 (size=12001) 2024-11-16T05:40:08,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/d3a1c0fbfc354e9094b5efa972ece3e0 2024-11-16T05:40:08,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/15dc7b60cfea4b159c98f557dcd58277 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/15dc7b60cfea4b159c98f557dcd58277 2024-11-16T05:40:08,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/15dc7b60cfea4b159c98f557dcd58277, entries=150, sequenceid=17, filesize=30.2 K 2024-11-16T05:40:08,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/605797a240684449a029c27ac5354808 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/605797a240684449a029c27ac5354808 2024-11-16T05:40:08,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/605797a240684449a029c27ac5354808, entries=150, sequenceid=17, filesize=11.7 K 2024-11-16T05:40:08,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/d3a1c0fbfc354e9094b5efa972ece3e0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/d3a1c0fbfc354e9094b5efa972ece3e0 2024-11-16T05:40:08,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/d3a1c0fbfc354e9094b5efa972ece3e0, entries=150, sequenceid=17, filesize=11.7 K 2024-11-16T05:40:08,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 82b1d437a2b8fce4a136c21be6257221 in 726ms, sequenceid=17, compaction requested=false 2024-11-16T05:40:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:08,337 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:08,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T05:40:08,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:40:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:08,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163200aad208c247a1b2731e691d2e4018_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735607677/Put/seqid=0 2024-11-16T05:40:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735668350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735668354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735668357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735668362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735668362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741964_1140 (size=12154) 2024-11-16T05:40:08,386 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:08,395 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163200aad208c247a1b2731e691d2e4018_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163200aad208c247a1b2731e691d2e4018_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:08,398 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3232ca2bbd1e42c8b23c0904e9feea73, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:08,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3232ca2bbd1e42c8b23c0904e9feea73 is 175, key is test_row_0/A:col10/1731735607677/Put/seqid=0 2024-11-16T05:40:08,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741965_1141 (size=30955) 2024-11-16T05:40:08,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3232ca2bbd1e42c8b23c0904e9feea73 2024-11-16T05:40:08,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/53b19a4ab29f4a7b819153f36a205f5e is 50, key is test_row_0/B:col10/1731735607677/Put/seqid=0 2024-11-16T05:40:08,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735668469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735668469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735668469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735668470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735668470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741966_1142 (size=12001) 2024-11-16T05:40:08,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/53b19a4ab29f4a7b819153f36a205f5e 2024-11-16T05:40:08,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:40:08,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:08,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:08,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:08,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:08,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/f9c3bb6dec734e99919b4af44e13c3ac is 50, key is test_row_0/C:col10/1731735607677/Put/seqid=0 2024-11-16T05:40:08,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741967_1143 (size=12001) 2024-11-16T05:40:08,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/f9c3bb6dec734e99919b4af44e13c3ac 2024-11-16T05:40:08,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3232ca2bbd1e42c8b23c0904e9feea73 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3232ca2bbd1e42c8b23c0904e9feea73 2024-11-16T05:40:08,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3232ca2bbd1e42c8b23c0904e9feea73, entries=150, sequenceid=42, filesize=30.2 K 2024-11-16T05:40:08,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/53b19a4ab29f4a7b819153f36a205f5e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/53b19a4ab29f4a7b819153f36a205f5e 2024-11-16T05:40:08,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/53b19a4ab29f4a7b819153f36a205f5e, entries=150, sequenceid=42, filesize=11.7 K 2024-11-16T05:40:08,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/f9c3bb6dec734e99919b4af44e13c3ac as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f9c3bb6dec734e99919b4af44e13c3ac 2024-11-16T05:40:08,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f9c3bb6dec734e99919b4af44e13c3ac, entries=150, sequenceid=42, filesize=11.7 K 2024-11-16T05:40:08,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 82b1d437a2b8fce4a136c21be6257221 in 255ms, sequenceid=42, compaction requested=false 2024-11-16T05:40:08,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:08,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:40:08,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:08,666 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T05:40:08,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166efd92ad3e87400887bc9aa37fa10ae5_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735608360/Put/seqid=0 2024-11-16T05:40:08,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:40:08,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741968_1144 (size=12154) 2024-11-16T05:40:08,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735668725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735668725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735668729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735668731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735668734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735668838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735668838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735668848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735668849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735668849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735669048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735669049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735669058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735669058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735669058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:09,130 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166efd92ad3e87400887bc9aa37fa10ae5_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166efd92ad3e87400887bc9aa37fa10ae5_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:09,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/a40e057451f04fab85062ead9f32a49c, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:09,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/a40e057451f04fab85062ead9f32a49c is 175, key is test_row_0/A:col10/1731735608360/Put/seqid=0 2024-11-16T05:40:09,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741969_1145 (size=30955) 2024-11-16T05:40:09,154 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/a40e057451f04fab85062ead9f32a49c 2024-11-16T05:40:09,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/054cce3227634879bcea95a687bb20b1 is 50, key is test_row_0/B:col10/1731735608360/Put/seqid=0 2024-11-16T05:40:09,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741970_1146 (size=12001) 2024-11-16T05:40:09,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735669358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735669358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735669368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735669368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735669368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,620 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/054cce3227634879bcea95a687bb20b1 2024-11-16T05:40:09,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/9e80331dfb6d4065aa811531463efdd2 is 50, key is test_row_0/C:col10/1731735608360/Put/seqid=0 2024-11-16T05:40:09,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741971_1147 (size=12001) 2024-11-16T05:40:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:40:09,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735669869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735669869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735669878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735669879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735669879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:10,042 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/9e80331dfb6d4065aa811531463efdd2 2024-11-16T05:40:10,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/a40e057451f04fab85062ead9f32a49c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a40e057451f04fab85062ead9f32a49c 2024-11-16T05:40:10,058 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a40e057451f04fab85062ead9f32a49c, entries=150, sequenceid=54, filesize=30.2 K 2024-11-16T05:40:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/054cce3227634879bcea95a687bb20b1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/054cce3227634879bcea95a687bb20b1 2024-11-16T05:40:10,070 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/054cce3227634879bcea95a687bb20b1, entries=150, sequenceid=54, filesize=11.7 K 2024-11-16T05:40:10,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/9e80331dfb6d4065aa811531463efdd2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9e80331dfb6d4065aa811531463efdd2 2024-11-16T05:40:10,077 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9e80331dfb6d4065aa811531463efdd2, entries=150, sequenceid=54, filesize=11.7 K 2024-11-16T05:40:10,079 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 82b1d437a2b8fce4a136c21be6257221 in 1413ms, sequenceid=54, compaction requested=true 2024-11-16T05:40:10,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:10,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:10,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-16T05:40:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-16T05:40:10,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-16T05:40:10,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4980 sec 2024-11-16T05:40:10,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.5060 sec 2024-11-16T05:40:10,287 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:40:10,883 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:10,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T05:40:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:10,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167a8baad1540d4f67b3ac69e175ba3c45_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735608724/Put/seqid=0 2024-11-16T05:40:10,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735670906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:10,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735670908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:10,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735670909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:10,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735670912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:10,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735670920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:10,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741972_1148 (size=14594) 2024-11-16T05:40:10,962 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:10,972 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167a8baad1540d4f67b3ac69e175ba3c45_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167a8baad1540d4f67b3ac69e175ba3c45_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:10,976 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/6cbac66200c44e7dbeb7655f58a438eb, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:10,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/6cbac66200c44e7dbeb7655f58a438eb is 175, key is test_row_0/A:col10/1731735608724/Put/seqid=0 2024-11-16T05:40:11,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735671029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741973_1149 (size=39549) 2024-11-16T05:40:11,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735671030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735671030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,032 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/6cbac66200c44e7dbeb7655f58a438eb 2024-11-16T05:40:11,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735671032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735671038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/5269d7e90a08404eb3a75574c638d9eb is 50, key is test_row_0/B:col10/1731735608724/Put/seqid=0 2024-11-16T05:40:11,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741974_1150 (size=12001) 2024-11-16T05:40:11,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/5269d7e90a08404eb3a75574c638d9eb 2024-11-16T05:40:11,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/f978c2da743145ef87884510662616e2 is 50, key is test_row_0/C:col10/1731735608724/Put/seqid=0 2024-11-16T05:40:11,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741975_1151 (size=12001) 2024-11-16T05:40:11,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/f978c2da743145ef87884510662616e2 2024-11-16T05:40:11,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/6cbac66200c44e7dbeb7655f58a438eb as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/6cbac66200c44e7dbeb7655f58a438eb 2024-11-16T05:40:11,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/6cbac66200c44e7dbeb7655f58a438eb, entries=200, sequenceid=80, filesize=38.6 K 2024-11-16T05:40:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/5269d7e90a08404eb3a75574c638d9eb as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/5269d7e90a08404eb3a75574c638d9eb 2024-11-16T05:40:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/5269d7e90a08404eb3a75574c638d9eb, entries=150, sequenceid=80, filesize=11.7 K 2024-11-16T05:40:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/f978c2da743145ef87884510662616e2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f978c2da743145ef87884510662616e2 2024-11-16T05:40:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f978c2da743145ef87884510662616e2, entries=150, sequenceid=80, filesize=11.7 K 2024-11-16T05:40:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 82b1d437a2b8fce4a136c21be6257221 in 276ms, sequenceid=80, compaction requested=true 2024-11-16T05:40:11,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:11,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:11,159 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:11,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:11,160 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,161 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:11,162 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/A is initiating minor compaction (all files) 2024-11-16T05:40:11,162 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/A in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,162 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/15dc7b60cfea4b159c98f557dcd58277, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3232ca2bbd1e42c8b23c0904e9feea73, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a40e057451f04fab85062ead9f32a49c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/6cbac66200c44e7dbeb7655f58a438eb] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=129.3 K 2024-11-16T05:40:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,162 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:11,162 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/15dc7b60cfea4b159c98f557dcd58277, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3232ca2bbd1e42c8b23c0904e9feea73, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a40e057451f04fab85062ead9f32a49c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/6cbac66200c44e7dbeb7655f58a438eb] 2024-11-16T05:40:11,163 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:11,163 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/B is initiating minor compaction (all files) 2024-11-16T05:40:11,163 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/B in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,164 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/605797a240684449a029c27ac5354808, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/53b19a4ab29f4a7b819153f36a205f5e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/054cce3227634879bcea95a687bb20b1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/5269d7e90a08404eb3a75574c638d9eb] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=46.9 K 2024-11-16T05:40:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,164 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15dc7b60cfea4b159c98f557dcd58277, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731735607592 2024-11-16T05:40:11,165 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 605797a240684449a029c27ac5354808, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731735607592 2024-11-16T05:40:11,165 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3232ca2bbd1e42c8b23c0904e9feea73, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731735607677 2024-11-16T05:40:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,166 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b19a4ab29f4a7b819153f36a205f5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731735607677 2024-11-16T05:40:11,166 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a40e057451f04fab85062ead9f32a49c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731735608346 2024-11-16T05:40:11,166 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 054cce3227634879bcea95a687bb20b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731735608346 2024-11-16T05:40:11,166 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cbac66200c44e7dbeb7655f58a438eb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731735608724 2024-11-16T05:40:11,167 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5269d7e90a08404eb3a75574c638d9eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731735608724 2024-11-16T05:40:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,190 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#B#compaction#132 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,191 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/a02e65c0d3d149dc93186cb0f368644f is 50, key is test_row_0/B:col10/1731735608724/Put/seqid=0 2024-11-16T05:40:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,198 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,218 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e92cd064870a476f815dbf809458bbe3_82b1d437a2b8fce4a136c21be6257221 store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741976_1152 (size=12139) 2024-11-16T05:40:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,227 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e92cd064870a476f815dbf809458bbe3_82b1d437a2b8fce4a136c21be6257221, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:11,227 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e92cd064870a476f815dbf809458bbe3_82b1d437a2b8fce4a136c21be6257221 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:11,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:11,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:40:11,252 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/a02e65c0d3d149dc93186cb0f368644f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/a02e65c0d3d149dc93186cb0f368644f 2024-11-16T05:40:11,260 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/B of 82b1d437a2b8fce4a136c21be6257221 into a02e65c0d3d149dc93186cb0f368644f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:11,260 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:11,261 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/B, priority=12, startTime=1731735611159; duration=0sec 2024-11-16T05:40:11,261 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:11,261 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:B 2024-11-16T05:40:11,261 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:11,262 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:11,262 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/C is initiating minor compaction (all files) 2024-11-16T05:40:11,262 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/C in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:11,263 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/d3a1c0fbfc354e9094b5efa972ece3e0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f9c3bb6dec734e99919b4af44e13c3ac, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9e80331dfb6d4065aa811531463efdd2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f978c2da743145ef87884510662616e2] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=46.9 K 2024-11-16T05:40:11,263 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting d3a1c0fbfc354e9094b5efa972ece3e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731735607592 2024-11-16T05:40:11,263 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f9c3bb6dec734e99919b4af44e13c3ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731735607677 2024-11-16T05:40:11,264 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e80331dfb6d4065aa811531463efdd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731735608346 2024-11-16T05:40:11,265 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f978c2da743145ef87884510662616e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731735608724 2024-11-16T05:40:11,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741977_1153 (size=4469) 2024-11-16T05:40:11,291 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#C#compaction#134 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:11,291 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/2cf152243af94cdc8a77aa7c8b129612 is 50, key is test_row_0/C:col10/1731735608724/Put/seqid=0 2024-11-16T05:40:11,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ff6aec366fde40ae9f2f5979e7f2c3fa_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735611239/Put/seqid=0 2024-11-16T05:40:11,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741978_1154 (size=12139) 2024-11-16T05:40:11,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735671315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735671315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735671318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735671319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,328 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/2cf152243af94cdc8a77aa7c8b129612 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/2cf152243af94cdc8a77aa7c8b129612 2024-11-16T05:40:11,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735671320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,338 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/C of 82b1d437a2b8fce4a136c21be6257221 into 2cf152243af94cdc8a77aa7c8b129612(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:11,338 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:11,338 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/C, priority=12, startTime=1731735611160; duration=0sec 2024-11-16T05:40:11,338 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:11,338 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:C 2024-11-16T05:40:11,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741979_1155 (size=12154) 2024-11-16T05:40:11,343 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,349 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ff6aec366fde40ae9f2f5979e7f2c3fa_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ff6aec366fde40ae9f2f5979e7f2c3fa_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:11,351 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/51df5eb87daf476e86f578580bc472c8, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:11,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/51df5eb87daf476e86f578580bc472c8 is 175, key is test_row_0/A:col10/1731735611239/Put/seqid=0 2024-11-16T05:40:11,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741980_1156 (size=30951) 2024-11-16T05:40:11,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/51df5eb87daf476e86f578580bc472c8 2024-11-16T05:40:11,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/b736cea335234d73b169368cea2ec1ad is 50, key is test_row_0/B:col10/1731735611239/Put/seqid=0 2024-11-16T05:40:11,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741981_1157 (size=9657) 2024-11-16T05:40:11,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/b736cea335234d73b169368cea2ec1ad 2024-11-16T05:40:11,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/5d6de6f501dc43c79c4f5666446d8cbe is 50, key is test_row_0/C:col10/1731735611239/Put/seqid=0 2024-11-16T05:40:11,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741982_1158 (size=9657) 2024-11-16T05:40:11,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735671428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735671428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735671428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735671429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735671438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735671639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735671639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735671639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735671639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735671648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,681 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#A#compaction#133 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:11,682 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/7af93718e1b1483c987dab59b8c89b13 is 175, key is test_row_0/A:col10/1731735608724/Put/seqid=0 2024-11-16T05:40:11,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741983_1159 (size=31093) 2024-11-16T05:40:11,737 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/7af93718e1b1483c987dab59b8c89b13 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7af93718e1b1483c987dab59b8c89b13 2024-11-16T05:40:11,746 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/A of 82b1d437a2b8fce4a136c21be6257221 into 7af93718e1b1483c987dab59b8c89b13(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:11,747 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:11,747 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/A, priority=12, startTime=1731735611159; duration=0sec 2024-11-16T05:40:11,747 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:11,747 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:A 2024-11-16T05:40:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:40:11,749 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:11,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-16T05:40:11,755 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:40:11,757 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:11,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:11,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/5d6de6f501dc43c79c4f5666446d8cbe 2024-11-16T05:40:11,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/51df5eb87daf476e86f578580bc472c8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/51df5eb87daf476e86f578580bc472c8 2024-11-16T05:40:11,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/51df5eb87daf476e86f578580bc472c8, entries=150, sequenceid=91, filesize=30.2 K 2024-11-16T05:40:11,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/b736cea335234d73b169368cea2ec1ad as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/b736cea335234d73b169368cea2ec1ad 2024-11-16T05:40:11,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/b736cea335234d73b169368cea2ec1ad, entries=100, sequenceid=91, filesize=9.4 K 2024-11-16T05:40:11,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/5d6de6f501dc43c79c4f5666446d8cbe as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5d6de6f501dc43c79c4f5666446d8cbe 2024-11-16T05:40:11,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5d6de6f501dc43c79c4f5666446d8cbe, entries=100, sequenceid=91, filesize=9.4 K 2024-11-16T05:40:11,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 82b1d437a2b8fce4a136c21be6257221 in 622ms, sequenceid=91, compaction requested=false 2024-11-16T05:40:11,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:40:11,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T05:40:11,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:11,916 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T05:40:11,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116df9b45808cdc48f48f1ff9d8b5fe45cb_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735611313/Put/seqid=0 2024-11-16T05:40:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:11,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:11,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735671958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735671961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735671961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735671963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735671964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:11,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741984_1160 (size=12154) 2024-11-16T05:40:11,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:11,997 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116df9b45808cdc48f48f1ff9d8b5fe45cb_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116df9b45808cdc48f48f1ff9d8b5fe45cb_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/0276652e38a14b42816d7c29518a2444, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/0276652e38a14b42816d7c29518a2444 is 175, key is test_row_0/A:col10/1731735611313/Put/seqid=0 2024-11-16T05:40:12,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741985_1161 (size=30955) 2024-11-16T05:40:12,040 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/0276652e38a14b42816d7c29518a2444 2024-11-16T05:40:12,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/ec1ea3647ece4da1a20da6c77e214b0b is 50, key is test_row_0/B:col10/1731735611313/Put/seqid=0 2024-11-16T05:40:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:40:12,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735672079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735672079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735672081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735672081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741986_1162 (size=12001) 2024-11-16T05:40:12,101 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/ec1ea3647ece4da1a20da6c77e214b0b 2024-11-16T05:40:12,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/8a9d372ec1924b59b6bcbcb5db8cdd83 is 50, key is test_row_0/C:col10/1731735611313/Put/seqid=0 2024-11-16T05:40:12,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741987_1163 (size=12001) 2024-11-16T05:40:12,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735672288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735672288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735672288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735672298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:40:12,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735672468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,547 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/8a9d372ec1924b59b6bcbcb5db8cdd83 2024-11-16T05:40:12,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/0276652e38a14b42816d7c29518a2444 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0276652e38a14b42816d7c29518a2444 2024-11-16T05:40:12,565 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0276652e38a14b42816d7c29518a2444, entries=150, sequenceid=119, filesize=30.2 K 2024-11-16T05:40:12,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/ec1ea3647ece4da1a20da6c77e214b0b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/ec1ea3647ece4da1a20da6c77e214b0b 2024-11-16T05:40:12,576 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/ec1ea3647ece4da1a20da6c77e214b0b, entries=150, sequenceid=119, filesize=11.7 K 2024-11-16T05:40:12,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/8a9d372ec1924b59b6bcbcb5db8cdd83 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8a9d372ec1924b59b6bcbcb5db8cdd83 2024-11-16T05:40:12,586 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8a9d372ec1924b59b6bcbcb5db8cdd83, entries=150, sequenceid=119, filesize=11.7 K 2024-11-16T05:40:12,588 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 82b1d437a2b8fce4a136c21be6257221 in 671ms, sequenceid=119, compaction requested=true 2024-11-16T05:40:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:12,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:12,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-16T05:40:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-16T05:40:12,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-16T05:40:12,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 836 msec 2024-11-16T05:40:12,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 846 msec 2024-11-16T05:40:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:12,602 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:12,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T05:40:12,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b3690f85270548fd8458e3dfe82f6068_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735612598/Put/seqid=0 2024-11-16T05:40:12,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741988_1164 (size=17184) 2024-11-16T05:40:12,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:12,660 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b3690f85270548fd8458e3dfe82f6068_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b3690f85270548fd8458e3dfe82f6068_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:12,661 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/7ade7b77003f4452a6ea9ec1cd81ce7d, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:12,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/7ade7b77003f4452a6ea9ec1cd81ce7d is 175, key is test_row_0/A:col10/1731735612598/Put/seqid=0 2024-11-16T05:40:12,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735672655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735672657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735672662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735672662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741989_1165 (size=48289) 2024-11-16T05:40:12,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735672769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735672778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735672778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735672788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:40:12,899 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:12,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:12,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-16T05:40:12,902 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:40:12,903 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:12,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:12,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735672978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735672989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:12,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735672990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735672999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:40:13,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:40:13,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:13,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:13,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:13,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:13,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:13,087 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/7ade7b77003f4452a6ea9ec1cd81ce7d 2024-11-16T05:40:13,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/16d3ccd1257a424e92d57e883ae4eb90 is 50, key is test_row_0/B:col10/1731735612598/Put/seqid=0 2024-11-16T05:40:13,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741990_1166 (size=12101) 2024-11-16T05:40:13,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/16d3ccd1257a424e92d57e883ae4eb90 2024-11-16T05:40:13,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/f7393f26c6304c3796c8b80509632890 is 50, key is test_row_0/C:col10/1731735612598/Put/seqid=0 2024-11-16T05:40:13,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741991_1167 (size=12101) 2024-11-16T05:40:13,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/f7393f26c6304c3796c8b80509632890 2024-11-16T05:40:13,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/7ade7b77003f4452a6ea9ec1cd81ce7d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7ade7b77003f4452a6ea9ec1cd81ce7d 2024-11-16T05:40:13,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7ade7b77003f4452a6ea9ec1cd81ce7d, entries=250, sequenceid=132, filesize=47.2 K 2024-11-16T05:40:13,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/16d3ccd1257a424e92d57e883ae4eb90 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16d3ccd1257a424e92d57e883ae4eb90 2024-11-16T05:40:13,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16d3ccd1257a424e92d57e883ae4eb90, entries=150, sequenceid=132, filesize=11.8 K 2024-11-16T05:40:13,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/f7393f26c6304c3796c8b80509632890 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f7393f26c6304c3796c8b80509632890 2024-11-16T05:40:13,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f7393f26c6304c3796c8b80509632890, entries=150, sequenceid=132, filesize=11.8 K 2024-11-16T05:40:13,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 82b1d437a2b8fce4a136c21be6257221 in 602ms, sequenceid=132, compaction requested=true 2024-11-16T05:40:13,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:13,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:13,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:13,204 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:13,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:13,205 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:13,206 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45898 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:13,206 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141288 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:13,206 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/B is initiating minor compaction (all files) 2024-11-16T05:40:13,206 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/A is initiating minor compaction (all files) 2024-11-16T05:40:13,206 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/A in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:13,206 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/B in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:13,206 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/a02e65c0d3d149dc93186cb0f368644f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/b736cea335234d73b169368cea2ec1ad, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/ec1ea3647ece4da1a20da6c77e214b0b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16d3ccd1257a424e92d57e883ae4eb90] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=44.8 K 2024-11-16T05:40:13,206 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7af93718e1b1483c987dab59b8c89b13, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/51df5eb87daf476e86f578580bc472c8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0276652e38a14b42816d7c29518a2444, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7ade7b77003f4452a6ea9ec1cd81ce7d] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=138.0 K 2024-11-16T05:40:13,206 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:13,207 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7af93718e1b1483c987dab59b8c89b13, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/51df5eb87daf476e86f578580bc472c8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0276652e38a14b42816d7c29518a2444, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7ade7b77003f4452a6ea9ec1cd81ce7d] 2024-11-16T05:40:13,207 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting a02e65c0d3d149dc93186cb0f368644f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731735608724 2024-11-16T05:40:13,207 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting b736cea335234d73b169368cea2ec1ad, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731735610900 2024-11-16T05:40:13,207 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7af93718e1b1483c987dab59b8c89b13, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731735608724 2024-11-16T05:40:13,207 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ec1ea3647ece4da1a20da6c77e214b0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731735611313 2024-11-16T05:40:13,208 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51df5eb87daf476e86f578580bc472c8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731735610900 2024-11-16T05:40:13,208 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 16d3ccd1257a424e92d57e883ae4eb90, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731735611951 2024-11-16T05:40:13,208 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0276652e38a14b42816d7c29518a2444, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731735611313 2024-11-16T05:40:13,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:40:13,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:13,209 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ade7b77003f4452a6ea9ec1cd81ce7d, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731735611951 2024-11-16T05:40:13,209 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T05:40:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:40:13,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116729adfde791d453592da9d62d8bd8d45_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735612649/Put/seqid=0 2024-11-16T05:40:13,237 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:13,251 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#B#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:13,251 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/66406e886cbf4b7ca5ae80b3630917e1 is 50, key is test_row_0/B:col10/1731735612598/Put/seqid=0 2024-11-16T05:40:13,262 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411164eb03823640646dd8ece66f574191f31_82b1d437a2b8fce4a136c21be6257221 store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:13,265 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411164eb03823640646dd8ece66f574191f31_82b1d437a2b8fce4a136c21be6257221, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:13,265 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164eb03823640646dd8ece66f574191f31_82b1d437a2b8fce4a136c21be6257221 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:13,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741992_1168 (size=12304) 2024-11-16T05:40:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:13,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:13,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741993_1169 (size=12375) 2024-11-16T05:40:13,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741994_1170 (size=4469) 2024-11-16T05:40:13,309 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/66406e886cbf4b7ca5ae80b3630917e1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/66406e886cbf4b7ca5ae80b3630917e1 2024-11-16T05:40:13,318 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/B of 82b1d437a2b8fce4a136c21be6257221 into 66406e886cbf4b7ca5ae80b3630917e1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:13,318 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:13,318 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/B, priority=12, startTime=1731735613204; duration=0sec 2024-11-16T05:40:13,318 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:13,318 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:B 2024-11-16T05:40:13,318 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:13,320 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45898 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:13,320 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/C is initiating minor compaction (all files) 2024-11-16T05:40:13,320 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/C in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:13,320 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/2cf152243af94cdc8a77aa7c8b129612, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5d6de6f501dc43c79c4f5666446d8cbe, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8a9d372ec1924b59b6bcbcb5db8cdd83, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f7393f26c6304c3796c8b80509632890] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=44.8 K 2024-11-16T05:40:13,320 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf152243af94cdc8a77aa7c8b129612, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731735608724 2024-11-16T05:40:13,321 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d6de6f501dc43c79c4f5666446d8cbe, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731735610900 2024-11-16T05:40:13,321 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a9d372ec1924b59b6bcbcb5db8cdd83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731735611313 2024-11-16T05:40:13,321 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f7393f26c6304c3796c8b80509632890, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731735611951 2024-11-16T05:40:13,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735673316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735673316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735673318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735673329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,340 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:13,340 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/847b3562ab554e328248f71cfd2f1bf4 is 50, key is test_row_0/C:col10/1731735612598/Put/seqid=0 2024-11-16T05:40:13,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741995_1171 (size=12375) 2024-11-16T05:40:13,364 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/847b3562ab554e328248f71cfd2f1bf4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/847b3562ab554e328248f71cfd2f1bf4 2024-11-16T05:40:13,372 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/C of 82b1d437a2b8fce4a136c21be6257221 into 847b3562ab554e328248f71cfd2f1bf4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:13,372 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:13,372 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/C, priority=12, startTime=1731735613205; duration=0sec 2024-11-16T05:40:13,372 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:13,372 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:C 2024-11-16T05:40:13,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735673438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735673438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735673438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735673438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735673488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,495 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in 82b1d437a2b8fce4a136c21be6257221 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:40:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:40:13,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735673648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735673648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735673648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735673648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:13,679 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116729adfde791d453592da9d62d8bd8d45_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116729adfde791d453592da9d62d8bd8d45_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:13,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/765a2a1d52a342eeaf245204f0d74e15, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:13,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/765a2a1d52a342eeaf245204f0d74e15 is 175, key is test_row_0/A:col10/1731735612649/Put/seqid=0 2024-11-16T05:40:13,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741996_1172 (size=31105) 2024-11-16T05:40:13,695 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#A#compaction#145 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:13,696 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/0a43e4e2a6df4a0ba241040d39b56e54 is 175, key is test_row_0/A:col10/1731735612598/Put/seqid=0 2024-11-16T05:40:13,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741997_1173 (size=31329) 2024-11-16T05:40:13,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735673959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735673958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735673968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735673968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:40:14,086 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/765a2a1d52a342eeaf245204f0d74e15 2024-11-16T05:40:14,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/e0747da12e824b6aa7f7699c604a767f is 50, key is test_row_0/B:col10/1731735612649/Put/seqid=0 2024-11-16T05:40:14,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741998_1174 (size=12151) 2024-11-16T05:40:14,117 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/e0747da12e824b6aa7f7699c604a767f 2024-11-16T05:40:14,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/92f9ffada0de4b1fbbed5f706e5e74df is 50, key is test_row_0/C:col10/1731735612649/Put/seqid=0 2024-11-16T05:40:14,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741999_1175 (size=12151) 2024-11-16T05:40:14,142 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/92f9ffada0de4b1fbbed5f706e5e74df 2024-11-16T05:40:14,147 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/0a43e4e2a6df4a0ba241040d39b56e54 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0a43e4e2a6df4a0ba241040d39b56e54 2024-11-16T05:40:14,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/765a2a1d52a342eeaf245204f0d74e15 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/765a2a1d52a342eeaf245204f0d74e15 2024-11-16T05:40:14,152 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/A of 82b1d437a2b8fce4a136c21be6257221 into 0a43e4e2a6df4a0ba241040d39b56e54(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:14,153 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:14,153 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/A, priority=12, startTime=1731735613204; duration=0sec 2024-11-16T05:40:14,153 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:14,153 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:A 2024-11-16T05:40:14,154 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/765a2a1d52a342eeaf245204f0d74e15, entries=150, sequenceid=155, filesize=30.4 K 2024-11-16T05:40:14,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/e0747da12e824b6aa7f7699c604a767f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0747da12e824b6aa7f7699c604a767f 2024-11-16T05:40:14,159 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0747da12e824b6aa7f7699c604a767f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T05:40:14,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/92f9ffada0de4b1fbbed5f706e5e74df as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/92f9ffada0de4b1fbbed5f706e5e74df 2024-11-16T05:40:14,166 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/92f9ffada0de4b1fbbed5f706e5e74df, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T05:40:14,167 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 82b1d437a2b8fce4a136c21be6257221 in 958ms, sequenceid=155, compaction requested=false 2024-11-16T05:40:14,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:14,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:14,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-16T05:40:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-16T05:40:14,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-16T05:40:14,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2660 sec 2024-11-16T05:40:14,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.2710 sec 2024-11-16T05:40:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:14,480 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:14,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T05:40:14,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ad20812dd4664c7cb912d14839362abd_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735613327/Put/seqid=0 2024-11-16T05:40:14,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735674525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735674526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735674528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735674528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742000_1176 (size=14794) 2024-11-16T05:40:14,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735674639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735674639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735674648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735674648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735674848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735674848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735674858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735674858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:14,949 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:40:14,949 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T05:40:14,950 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:14,969 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ad20812dd4664c7cb912d14839362abd_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ad20812dd4664c7cb912d14839362abd_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:14,970 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/9109ef51b2b24d0ca2ae94d878acb3d8, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:14,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/9109ef51b2b24d0ca2ae94d878acb3d8 is 175, key is test_row_0/A:col10/1731735613327/Put/seqid=0 2024-11-16T05:40:15,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742001_1177 (size=39749) 2024-11-16T05:40:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:40:15,049 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:15,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-16T05:40:15,053 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:15,054 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:15,055 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:40:15,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735675159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735675159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:40:15,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735675168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735675168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T05:40:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:15,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T05:40:15,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:15,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:15,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:40:15,404 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/9109ef51b2b24d0ca2ae94d878acb3d8 2024-11-16T05:40:15,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/9463b073e1944319b159bcc095d90c8d is 50, key is test_row_0/B:col10/1731735613327/Put/seqid=0 2024-11-16T05:40:15,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742002_1178 (size=12151) 2024-11-16T05:40:15,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/9463b073e1944319b159bcc095d90c8d 2024-11-16T05:40:15,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/dda285d57a6d4aef8a4ce53f63d9e6ed is 50, key is test_row_0/C:col10/1731735613327/Put/seqid=0 2024-11-16T05:40:15,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735675508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,512 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in 82b1d437a2b8fce4a136c21be6257221 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:40:15,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T05:40:15,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:15,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:15,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:15,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742003_1179 (size=12151) 2024-11-16T05:40:15,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/dda285d57a6d4aef8a4ce53f63d9e6ed 2024-11-16T05:40:15,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/9109ef51b2b24d0ca2ae94d878acb3d8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/9109ef51b2b24d0ca2ae94d878acb3d8 2024-11-16T05:40:15,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/9109ef51b2b24d0ca2ae94d878acb3d8, entries=200, sequenceid=172, filesize=38.8 K 2024-11-16T05:40:15,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/9463b073e1944319b159bcc095d90c8d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/9463b073e1944319b159bcc095d90c8d 2024-11-16T05:40:15,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/9463b073e1944319b159bcc095d90c8d, entries=150, sequenceid=172, filesize=11.9 K 2024-11-16T05:40:15,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/dda285d57a6d4aef8a4ce53f63d9e6ed as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/dda285d57a6d4aef8a4ce53f63d9e6ed 2024-11-16T05:40:15,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/dda285d57a6d4aef8a4ce53f63d9e6ed, entries=150, sequenceid=172, filesize=11.9 K 2024-11-16T05:40:15,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 82b1d437a2b8fce4a136c21be6257221 in 1080ms, sequenceid=172, compaction requested=true 2024-11-16T05:40:15,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:15,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:15,561 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:15,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:15,561 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:15,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:15,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:15,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:15,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:15,562 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:15,562 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:15,562 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/A is initiating minor compaction (all files) 2024-11-16T05:40:15,562 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/B is initiating minor compaction (all files) 2024-11-16T05:40:15,562 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/B in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,562 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/A in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,562 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/66406e886cbf4b7ca5ae80b3630917e1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0747da12e824b6aa7f7699c604a767f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/9463b073e1944319b159bcc095d90c8d] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=35.8 K 2024-11-16T05:40:15,562 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0a43e4e2a6df4a0ba241040d39b56e54, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/765a2a1d52a342eeaf245204f0d74e15, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/9109ef51b2b24d0ca2ae94d878acb3d8] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=99.8 K 2024-11-16T05:40:15,563 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,563 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0a43e4e2a6df4a0ba241040d39b56e54, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/765a2a1d52a342eeaf245204f0d74e15, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/9109ef51b2b24d0ca2ae94d878acb3d8] 2024-11-16T05:40:15,563 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 66406e886cbf4b7ca5ae80b3630917e1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731735611951 2024-11-16T05:40:15,563 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a43e4e2a6df4a0ba241040d39b56e54, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731735611951 2024-11-16T05:40:15,563 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e0747da12e824b6aa7f7699c604a767f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731735612649 2024-11-16T05:40:15,564 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 765a2a1d52a342eeaf245204f0d74e15, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731735612649 2024-11-16T05:40:15,564 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9463b073e1944319b159bcc095d90c8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731735613314 2024-11-16T05:40:15,564 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9109ef51b2b24d0ca2ae94d878acb3d8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731735613314 2024-11-16T05:40:15,582 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:15,599 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:15,600 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/0b7629a77bc54a468f7fa23c09d283ab is 50, key is test_row_0/B:col10/1731735613327/Put/seqid=0 2024-11-16T05:40:15,606 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116fffa28592741449b9e1e147d41c977a9_82b1d437a2b8fce4a136c21be6257221 store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:15,608 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116fffa28592741449b9e1e147d41c977a9_82b1d437a2b8fce4a136c21be6257221, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:15,608 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fffa28592741449b9e1e147d41c977a9_82b1d437a2b8fce4a136c21be6257221 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:15,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742004_1180 (size=12527) 2024-11-16T05:40:15,635 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/0b7629a77bc54a468f7fa23c09d283ab as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/0b7629a77bc54a468f7fa23c09d283ab 2024-11-16T05:40:15,642 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/B of 82b1d437a2b8fce4a136c21be6257221 into 0b7629a77bc54a468f7fa23c09d283ab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:15,642 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:15,642 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/B, priority=13, startTime=1731735615561; duration=0sec 2024-11-16T05:40:15,642 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:15,642 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:B 2024-11-16T05:40:15,642 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:15,644 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:15,644 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/C is initiating minor compaction (all files) 2024-11-16T05:40:15,644 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/C in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,644 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/847b3562ab554e328248f71cfd2f1bf4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/92f9ffada0de4b1fbbed5f706e5e74df, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/dda285d57a6d4aef8a4ce53f63d9e6ed] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=35.8 K 2024-11-16T05:40:15,645 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 847b3562ab554e328248f71cfd2f1bf4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731735611951 2024-11-16T05:40:15,645 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f9ffada0de4b1fbbed5f706e5e74df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731735612649 2024-11-16T05:40:15,646 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting dda285d57a6d4aef8a4ce53f63d9e6ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731735613314 2024-11-16T05:40:15,657 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:15,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742005_1181 (size=4469) 2024-11-16T05:40:15,657 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/01523a38e3234f62950f269f8441ae86 is 50, key is test_row_0/C:col10/1731735613327/Put/seqid=0 2024-11-16T05:40:15,664 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#A#compaction#153 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:15,665 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/d3f52d8cd3b7479fa3b586d8e3ff9869 is 175, key is test_row_0/A:col10/1731735613327/Put/seqid=0 2024-11-16T05:40:15,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T05:40:15,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:15,676 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T05:40:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:15,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:40:15,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742006_1182 (size=12527) 2024-11-16T05:40:15,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735675698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735675704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735675706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735675706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f00a9fdbecc64468a5131c5e535b5bf9_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735614514/Put/seqid=0 2024-11-16T05:40:15,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742007_1183 (size=31481) 2024-11-16T05:40:15,740 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/d3f52d8cd3b7479fa3b586d8e3ff9869 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/d3f52d8cd3b7479fa3b586d8e3ff9869 2024-11-16T05:40:15,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742008_1184 (size=12304) 2024-11-16T05:40:15,748 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/A of 82b1d437a2b8fce4a136c21be6257221 into d3f52d8cd3b7479fa3b586d8e3ff9869(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:15,748 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:15,748 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/A, priority=13, startTime=1731735615560; duration=0sec 2024-11-16T05:40:15,748 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:15,748 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:A 2024-11-16T05:40:15,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735675808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735675818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735675819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735675819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735676018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735676028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735676028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735676028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,110 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/01523a38e3234f62950f269f8441ae86 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/01523a38e3234f62950f269f8441ae86 2024-11-16T05:40:16,120 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/C of 82b1d437a2b8fce4a136c21be6257221 into 01523a38e3234f62950f269f8441ae86(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:16,120 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:16,120 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/C, priority=13, startTime=1731735615561; duration=0sec 2024-11-16T05:40:16,120 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:16,120 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:C 2024-11-16T05:40:16,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:16,150 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f00a9fdbecc64468a5131c5e535b5bf9_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f00a9fdbecc64468a5131c5e535b5bf9_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:16,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/27d229003f8d47d0b7d63838a64adb92, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:16,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/27d229003f8d47d0b7d63838a64adb92 is 175, key is test_row_0/A:col10/1731735614514/Put/seqid=0 2024-11-16T05:40:16,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742009_1185 (size=31105) 2024-11-16T05:40:16,158 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/27d229003f8d47d0b7d63838a64adb92 2024-11-16T05:40:16,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/54865ad5eeb04bc69dc4ae9dd0748e2c is 50, key is test_row_0/B:col10/1731735614514/Put/seqid=0 2024-11-16T05:40:16,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742010_1186 (size=12151) 2024-11-16T05:40:16,190 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/54865ad5eeb04bc69dc4ae9dd0748e2c 2024-11-16T05:40:16,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/8cc52cf81f3c4b56a314b019e2fdc2b4 is 50, key is test_row_0/C:col10/1731735614514/Put/seqid=0 2024-11-16T05:40:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:40:16,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742011_1187 (size=12151) 2024-11-16T05:40:16,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735676328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735676338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735676339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735676339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,614 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/8cc52cf81f3c4b56a314b019e2fdc2b4 2024-11-16T05:40:16,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/27d229003f8d47d0b7d63838a64adb92 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/27d229003f8d47d0b7d63838a64adb92 2024-11-16T05:40:16,626 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/27d229003f8d47d0b7d63838a64adb92, entries=150, sequenceid=196, filesize=30.4 K 2024-11-16T05:40:16,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/54865ad5eeb04bc69dc4ae9dd0748e2c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/54865ad5eeb04bc69dc4ae9dd0748e2c 2024-11-16T05:40:16,636 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/54865ad5eeb04bc69dc4ae9dd0748e2c, entries=150, sequenceid=196, filesize=11.9 K 2024-11-16T05:40:16,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/8cc52cf81f3c4b56a314b019e2fdc2b4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8cc52cf81f3c4b56a314b019e2fdc2b4 2024-11-16T05:40:16,656 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8cc52cf81f3c4b56a314b019e2fdc2b4, entries=150, sequenceid=196, filesize=11.9 K 2024-11-16T05:40:16,657 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 82b1d437a2b8fce4a136c21be6257221 in 981ms, sequenceid=196, compaction requested=false 2024-11-16T05:40:16,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:16,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:16,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-16T05:40:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-16T05:40:16,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-16T05:40:16,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6040 sec 2024-11-16T05:40:16,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6120 sec 2024-11-16T05:40:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:16,843 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:16,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T05:40:16,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111663386aff40e34366811fd067b637be26_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735615703/Put/seqid=0 2024-11-16T05:40:16,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742012_1188 (size=14794) 2024-11-16T05:40:16,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735676885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735676889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:16,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735676892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735676892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:16,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111663386aff40e34366811fd067b637be26_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111663386aff40e34366811fd067b637be26_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:16,901 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/74195d8290e84110aa818323ba038b35, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:16,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/74195d8290e84110aa818323ba038b35 is 175, key is test_row_0/A:col10/1731735615703/Put/seqid=0 2024-11-16T05:40:16,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742013_1189 (size=39749) 2024-11-16T05:40:16,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735676998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735677009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735677009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735677009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:40:17,209 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:17,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-16T05:40:17,212 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T05:40:17,213 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:17,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:17,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735677208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735677220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735677220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735677220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T05:40:17,344 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/74195d8290e84110aa818323ba038b35 2024-11-16T05:40:17,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/c9349eb043de44d08cfd3a7344fb823c is 50, key is test_row_0/B:col10/1731735615703/Put/seqid=0 2024-11-16T05:40:17,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T05:40:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742014_1190 (size=12151) 2024-11-16T05:40:17,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735677519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T05:40:17,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:17,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T05:40:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735677547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735677547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735677548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:17,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T05:40:17,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:17,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/c9349eb043de44d08cfd3a7344fb823c 2024-11-16T05:40:17,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/652c4c7b4a6349ccbca3d7c36949478a is 50, key is test_row_0/C:col10/1731735615703/Put/seqid=0 2024-11-16T05:40:17,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742015_1191 (size=12151) 2024-11-16T05:40:17,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/652c4c7b4a6349ccbca3d7c36949478a 2024-11-16T05:40:17,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/74195d8290e84110aa818323ba038b35 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/74195d8290e84110aa818323ba038b35 2024-11-16T05:40:17,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T05:40:17,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:17,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/74195d8290e84110aa818323ba038b35, entries=200, sequenceid=213, filesize=38.8 K 2024-11-16T05:40:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T05:40:17,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/c9349eb043de44d08cfd3a7344fb823c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/c9349eb043de44d08cfd3a7344fb823c 2024-11-16T05:40:17,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/c9349eb043de44d08cfd3a7344fb823c, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T05:40:17,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/652c4c7b4a6349ccbca3d7c36949478a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/652c4c7b4a6349ccbca3d7c36949478a 2024-11-16T05:40:17,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/652c4c7b4a6349ccbca3d7c36949478a, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T05:40:17,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 82b1d437a2b8fce4a136c21be6257221 in 1038ms, sequenceid=213, compaction requested=true 2024-11-16T05:40:17,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:17,881 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:17,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:17,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:17,882 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:17,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:17,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:17,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:17,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:17,883 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:17,883 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/A is initiating minor compaction (all files) 2024-11-16T05:40:17,883 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/A in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,883 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/d3f52d8cd3b7479fa3b586d8e3ff9869, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/27d229003f8d47d0b7d63838a64adb92, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/74195d8290e84110aa818323ba038b35] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=99.9 K 2024-11-16T05:40:17,883 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,883 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/d3f52d8cd3b7479fa3b586d8e3ff9869, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/27d229003f8d47d0b7d63838a64adb92, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/74195d8290e84110aa818323ba038b35] 2024-11-16T05:40:17,884 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f52d8cd3b7479fa3b586d8e3ff9869, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731735613314 2024-11-16T05:40:17,884 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27d229003f8d47d0b7d63838a64adb92, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731735614514 2024-11-16T05:40:17,884 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:17,884 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/B is initiating minor compaction (all files) 2024-11-16T05:40:17,884 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/B in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:17,885 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/0b7629a77bc54a468f7fa23c09d283ab, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/54865ad5eeb04bc69dc4ae9dd0748e2c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/c9349eb043de44d08cfd3a7344fb823c] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=36.0 K 2024-11-16T05:40:17,885 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74195d8290e84110aa818323ba038b35, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731735615694 2024-11-16T05:40:17,885 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7629a77bc54a468f7fa23c09d283ab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731735613314 2024-11-16T05:40:17,885 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 54865ad5eeb04bc69dc4ae9dd0748e2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731735614514 2024-11-16T05:40:17,886 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting c9349eb043de44d08cfd3a7344fb823c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731735615703 2024-11-16T05:40:17,895 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:17,895 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/f227b1c455564ad19c04f472b1278655 is 50, key is test_row_0/B:col10/1731735615703/Put/seqid=0 2024-11-16T05:40:17,898 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:17,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742016_1192 (size=12629) 2024-11-16T05:40:17,922 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116a558b411d964492e870b963ab211bad9_82b1d437a2b8fce4a136c21be6257221 store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:17,924 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116a558b411d964492e870b963ab211bad9_82b1d437a2b8fce4a136c21be6257221, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:17,924 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a558b411d964492e870b963ab211bad9_82b1d437a2b8fce4a136c21be6257221 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:17,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742017_1193 (size=4469) 2024-11-16T05:40:18,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T05:40:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:18,009 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T05:40:18,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161003b4bf8125451d9fe8da3e431e53ca_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735616888/Put/seqid=0 2024-11-16T05:40:18,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:18,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742018_1194 (size=12304) 2024-11-16T05:40:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:18,046 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161003b4bf8125451d9fe8da3e431e53ca_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161003b4bf8125451d9fe8da3e431e53ca_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/b8e3e607d0c54bb79deff32cc6d57c61, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:18,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/b8e3e607d0c54bb79deff32cc6d57c61 is 175, key is test_row_0/A:col10/1731735616888/Put/seqid=0 2024-11-16T05:40:18,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742019_1195 (size=31105) 2024-11-16T05:40:18,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735678068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735678069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735678071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735678073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735678178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735678178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735678178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,323 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/f227b1c455564ad19c04f472b1278655 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f227b1c455564ad19c04f472b1278655 2024-11-16T05:40:18,330 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/B of 82b1d437a2b8fce4a136c21be6257221 into f227b1c455564ad19c04f472b1278655(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:18,330 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:18,330 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/B, priority=13, startTime=1731735617882; duration=0sec 2024-11-16T05:40:18,330 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:18,330 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:B 2024-11-16T05:40:18,330 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:18,332 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:18,332 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/C is initiating minor compaction (all files) 2024-11-16T05:40:18,332 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/C in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:18,332 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/01523a38e3234f62950f269f8441ae86, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8cc52cf81f3c4b56a314b019e2fdc2b4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/652c4c7b4a6349ccbca3d7c36949478a] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=36.0 K 2024-11-16T05:40:18,333 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 01523a38e3234f62950f269f8441ae86, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731735613314 2024-11-16T05:40:18,333 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc52cf81f3c4b56a314b019e2fdc2b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731735614514 2024-11-16T05:40:18,334 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 652c4c7b4a6349ccbca3d7c36949478a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731735615703 2024-11-16T05:40:18,337 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#A#compaction#163 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:18,337 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/a56a28545dd8452fa72cbf98b9acd2a8 is 175, key is test_row_0/A:col10/1731735615703/Put/seqid=0 2024-11-16T05:40:18,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742020_1196 (size=31583) 2024-11-16T05:40:18,349 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:18,350 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/7ec35e8782f049f1baf5999656f9f5d2 is 50, key is test_row_0/C:col10/1731735615703/Put/seqid=0 2024-11-16T05:40:18,358 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/a56a28545dd8452fa72cbf98b9acd2a8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a56a28545dd8452fa72cbf98b9acd2a8 2024-11-16T05:40:18,365 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/A of 82b1d437a2b8fce4a136c21be6257221 into a56a28545dd8452fa72cbf98b9acd2a8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:18,365 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:18,365 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/A, priority=13, startTime=1731735617881; duration=0sec 2024-11-16T05:40:18,366 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:18,366 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:A 2024-11-16T05:40:18,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742021_1197 (size=12629) 2024-11-16T05:40:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T05:40:18,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735678388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735678388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735678388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,456 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/b8e3e607d0c54bb79deff32cc6d57c61 2024-11-16T05:40:18,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/e34a407f7f2e4ff79ed1b581d2edd9a6 is 50, key is test_row_0/B:col10/1731735616888/Put/seqid=0 2024-11-16T05:40:18,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742022_1198 (size=12151) 2024-11-16T05:40:18,488 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/e34a407f7f2e4ff79ed1b581d2edd9a6 2024-11-16T05:40:18,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/89289f095fd545b1a5f89cc516d4acbe is 50, key is test_row_0/C:col10/1731735616888/Put/seqid=0 2024-11-16T05:40:18,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742023_1199 (size=12151) 2024-11-16T05:40:18,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735678698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735678698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735678698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:18,775 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/7ec35e8782f049f1baf5999656f9f5d2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/7ec35e8782f049f1baf5999656f9f5d2 2024-11-16T05:40:18,784 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/C of 82b1d437a2b8fce4a136c21be6257221 into 7ec35e8782f049f1baf5999656f9f5d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:18,784 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:18,784 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/C, priority=13, startTime=1731735617882; duration=0sec 2024-11-16T05:40:18,784 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:18,785 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:C 2024-11-16T05:40:18,941 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/89289f095fd545b1a5f89cc516d4acbe 2024-11-16T05:40:18,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/b8e3e607d0c54bb79deff32cc6d57c61 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b8e3e607d0c54bb79deff32cc6d57c61 2024-11-16T05:40:18,955 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b8e3e607d0c54bb79deff32cc6d57c61, entries=150, sequenceid=234, filesize=30.4 K 2024-11-16T05:40:18,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/e34a407f7f2e4ff79ed1b581d2edd9a6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e34a407f7f2e4ff79ed1b581d2edd9a6 2024-11-16T05:40:18,965 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e34a407f7f2e4ff79ed1b581d2edd9a6, entries=150, sequenceid=234, filesize=11.9 K 2024-11-16T05:40:18,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/89289f095fd545b1a5f89cc516d4acbe as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/89289f095fd545b1a5f89cc516d4acbe 2024-11-16T05:40:18,974 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/89289f095fd545b1a5f89cc516d4acbe, entries=150, sequenceid=234, filesize=11.9 K 2024-11-16T05:40:18,976 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 82b1d437a2b8fce4a136c21be6257221 in 967ms, sequenceid=234, compaction requested=false 2024-11-16T05:40:18,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:18,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:18,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-16T05:40:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-16T05:40:18,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-16T05:40:18,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7640 sec 2024-11-16T05:40:18,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.7710 sec 2024-11-16T05:40:19,091 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:19,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T05:40:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:19,109 INFO [master/3456ee6a3164:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T05:40:19,109 INFO [master/3456ee6a3164:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T05:40:19,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f50e59bf893d4353bf4d273a703568c5_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735619089/Put/seqid=0 2024-11-16T05:40:19,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742024_1200 (size=14794) 2024-11-16T05:40:19,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:19,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f50e59bf893d4353bf4d273a703568c5_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f50e59bf893d4353bf4d273a703568c5_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:19,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3fcce2754cb8427c9cfd60b8d7bc27cd, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:19,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3fcce2754cb8427c9cfd60b8d7bc27cd is 175, key is test_row_0/A:col10/1731735619089/Put/seqid=0 2024-11-16T05:40:19,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742025_1201 (size=39749) 2024-11-16T05:40:19,211 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3fcce2754cb8427c9cfd60b8d7bc27cd 2024-11-16T05:40:19,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/f394a9289ba948eb861895568372e8d2 is 50, key is test_row_0/B:col10/1731735619089/Put/seqid=0 2024-11-16T05:40:19,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742026_1202 (size=12151) 2024-11-16T05:40:19,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735679234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735679236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735679242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735679242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735679349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735679349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735679358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735679358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T05:40:19,378 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:19,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-16T05:40:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T05:40:19,383 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:19,384 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:19,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T05:40:19,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55396 deadline: 1731735679538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,541 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in 82b1d437a2b8fce4a136c21be6257221 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:40:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T05:40:19,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:19,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:19,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:19,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:19,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:19,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735679558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735679559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735679568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735679568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/f394a9289ba948eb861895568372e8d2 2024-11-16T05:40:19,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/9f3b2f785c0540579c36dfe6ef665f29 is 50, key is test_row_0/C:col10/1731735619089/Put/seqid=0 2024-11-16T05:40:19,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742027_1203 (size=12151) 2024-11-16T05:40:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T05:40:19,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T05:40:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:19,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:19,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T05:40:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:19,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:19,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:19,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735679868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735679868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735679878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735679878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T05:40:20,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T05:40:20,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:20,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:20,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:20,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:20,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:20,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/9f3b2f785c0540579c36dfe6ef665f29 2024-11-16T05:40:20,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3fcce2754cb8427c9cfd60b8d7bc27cd as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3fcce2754cb8427c9cfd60b8d7bc27cd 2024-11-16T05:40:20,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3fcce2754cb8427c9cfd60b8d7bc27cd, entries=200, sequenceid=253, filesize=38.8 K 2024-11-16T05:40:20,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/f394a9289ba948eb861895568372e8d2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f394a9289ba948eb861895568372e8d2 2024-11-16T05:40:20,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f394a9289ba948eb861895568372e8d2, entries=150, sequenceid=253, filesize=11.9 K 2024-11-16T05:40:20,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/9f3b2f785c0540579c36dfe6ef665f29 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9f3b2f785c0540579c36dfe6ef665f29 2024-11-16T05:40:20,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9f3b2f785c0540579c36dfe6ef665f29, entries=150, sequenceid=253, filesize=11.9 K 2024-11-16T05:40:20,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 82b1d437a2b8fce4a136c21be6257221 in 1012ms, sequenceid=253, compaction requested=true 2024-11-16T05:40:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:20,103 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:20,103 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:20,104 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:20,104 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:20,104 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/B is initiating minor compaction (all files) 2024-11-16T05:40:20,104 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/A is initiating minor compaction (all files) 2024-11-16T05:40:20,104 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/A in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:20,104 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/B in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:20,104 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a56a28545dd8452fa72cbf98b9acd2a8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b8e3e607d0c54bb79deff32cc6d57c61, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3fcce2754cb8427c9cfd60b8d7bc27cd] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=100.0 K 2024-11-16T05:40:20,104 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f227b1c455564ad19c04f472b1278655, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e34a407f7f2e4ff79ed1b581d2edd9a6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f394a9289ba948eb861895568372e8d2] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=36.1 K 2024-11-16T05:40:20,104 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:20,104 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a56a28545dd8452fa72cbf98b9acd2a8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b8e3e607d0c54bb79deff32cc6d57c61, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3fcce2754cb8427c9cfd60b8d7bc27cd] 2024-11-16T05:40:20,105 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f227b1c455564ad19c04f472b1278655, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731735615703 2024-11-16T05:40:20,105 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56a28545dd8452fa72cbf98b9acd2a8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731735615703 2024-11-16T05:40:20,105 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e34a407f7f2e4ff79ed1b581d2edd9a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731735616883 2024-11-16T05:40:20,105 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8e3e607d0c54bb79deff32cc6d57c61, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731735616883 2024-11-16T05:40:20,105 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f394a9289ba948eb861895568372e8d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731735618063 2024-11-16T05:40:20,106 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fcce2754cb8427c9cfd60b8d7bc27cd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731735618063 2024-11-16T05:40:20,114 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:20,115 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:20,115 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/400ca4f626a243b19c0bb19dab7ee344 is 50, key is test_row_0/B:col10/1731735619089/Put/seqid=0 2024-11-16T05:40:20,117 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411166e1f3c27de324b0087c576059ebbb8e2_82b1d437a2b8fce4a136c21be6257221 store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:20,119 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411166e1f3c27de324b0087c576059ebbb8e2_82b1d437a2b8fce4a136c21be6257221, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:20,119 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166e1f3c27de324b0087c576059ebbb8e2_82b1d437a2b8fce4a136c21be6257221 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:20,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742028_1204 (size=12731) 2024-11-16T05:40:20,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742029_1205 (size=4469) 2024-11-16T05:40:20,152 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/400ca4f626a243b19c0bb19dab7ee344 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/400ca4f626a243b19c0bb19dab7ee344 2024-11-16T05:40:20,158 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/B of 82b1d437a2b8fce4a136c21be6257221 into 400ca4f626a243b19c0bb19dab7ee344(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:20,158 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:20,158 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/B, priority=13, startTime=1731735620103; duration=0sec 2024-11-16T05:40:20,159 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:20,159 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:B 2024-11-16T05:40:20,159 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:20,161 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:20,161 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/C is initiating minor compaction (all files) 2024-11-16T05:40:20,161 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/C in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:20,161 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/7ec35e8782f049f1baf5999656f9f5d2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/89289f095fd545b1a5f89cc516d4acbe, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9f3b2f785c0540579c36dfe6ef665f29] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=36.1 K 2024-11-16T05:40:20,162 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ec35e8782f049f1baf5999656f9f5d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731735615703 2024-11-16T05:40:20,162 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 89289f095fd545b1a5f89cc516d4acbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731735616883 2024-11-16T05:40:20,162 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3b2f785c0540579c36dfe6ef665f29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731735618063 2024-11-16T05:40:20,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T05:40:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:20,165 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T05:40:20,184 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:20,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116880c63a76e1d444798802c9a42e160a6_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735619233/Put/seqid=0 2024-11-16T05:40:20,188 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/5858a96295ca41248d2925bb44179650 is 50, key is test_row_0/C:col10/1731735619089/Put/seqid=0 2024-11-16T05:40:20,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742030_1206 (size=12454) 2024-11-16T05:40:20,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742031_1207 (size=12731) 2024-11-16T05:40:20,223 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/5858a96295ca41248d2925bb44179650 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5858a96295ca41248d2925bb44179650 2024-11-16T05:40:20,238 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/C of 82b1d437a2b8fce4a136c21be6257221 into 5858a96295ca41248d2925bb44179650(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:20,238 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:20,238 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/C, priority=13, startTime=1731735620103; duration=0sec 2024-11-16T05:40:20,238 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:20,238 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:C 2024-11-16T05:40:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:20,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735680397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735680400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735680405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735680404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735680508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T05:40:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735680519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735680519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735680519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,543 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#A#compaction#172 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:20,544 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/38746f96170449e2861b398bb7d131dc is 175, key is test_row_0/A:col10/1731735619089/Put/seqid=0 2024-11-16T05:40:20,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742032_1208 (size=31685) 2024-11-16T05:40:20,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:20,623 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116880c63a76e1d444798802c9a42e160a6_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116880c63a76e1d444798802c9a42e160a6_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:20,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c978a4c093634ea6b832adc7544738c7, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c978a4c093634ea6b832adc7544738c7 is 175, key is test_row_0/A:col10/1731735619233/Put/seqid=0 2024-11-16T05:40:20,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742033_1209 (size=31255) 2024-11-16T05:40:20,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735680718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735680728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735680728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735680729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:20,992 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/38746f96170449e2861b398bb7d131dc as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/38746f96170449e2861b398bb7d131dc 2024-11-16T05:40:20,999 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/A of 82b1d437a2b8fce4a136c21be6257221 into 38746f96170449e2861b398bb7d131dc(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:20,999 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:20,999 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/A, priority=13, startTime=1731735620103; duration=0sec 2024-11-16T05:40:20,999 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:20,999 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:A 2024-11-16T05:40:21,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735681028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735681039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735681040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,042 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c978a4c093634ea6b832adc7544738c7 2024-11-16T05:40:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735681040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/aacd00aea3874767a1fbef280c589c8a is 50, key is test_row_0/B:col10/1731735619233/Put/seqid=0 2024-11-16T05:40:21,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742034_1210 (size=12301) 2024-11-16T05:40:21,492 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/aacd00aea3874767a1fbef280c589c8a 2024-11-16T05:40:21,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/27739c1bca9d4fd6a0799cd9315ab92f is 50, key is test_row_0/C:col10/1731735619233/Put/seqid=0 2024-11-16T05:40:21,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742035_1211 (size=12301) 2024-11-16T05:40:21,509 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/27739c1bca9d4fd6a0799cd9315ab92f 2024-11-16T05:40:21,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c978a4c093634ea6b832adc7544738c7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c978a4c093634ea6b832adc7544738c7 2024-11-16T05:40:21,520 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c978a4c093634ea6b832adc7544738c7, entries=150, sequenceid=274, filesize=30.5 K 2024-11-16T05:40:21,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/aacd00aea3874767a1fbef280c589c8a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/aacd00aea3874767a1fbef280c589c8a 2024-11-16T05:40:21,525 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/aacd00aea3874767a1fbef280c589c8a, entries=150, sequenceid=274, filesize=12.0 K 2024-11-16T05:40:21,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/27739c1bca9d4fd6a0799cd9315ab92f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/27739c1bca9d4fd6a0799cd9315ab92f 2024-11-16T05:40:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T05:40:21,532 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/27739c1bca9d4fd6a0799cd9315ab92f, entries=150, sequenceid=274, filesize=12.0 K 2024-11-16T05:40:21,533 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 82b1d437a2b8fce4a136c21be6257221 in 1368ms, sequenceid=274, compaction requested=false 2024-11-16T05:40:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-16T05:40:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-16T05:40:21,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-16T05:40:21,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1530 sec 2024-11-16T05:40:21,539 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:21,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T05:40:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:21,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.1610 sec 2024-11-16T05:40:21,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a5582e9ac2d6430aa11aca5efdd5dec4_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735620398/Put/seqid=0 2024-11-16T05:40:21,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742036_1212 (size=14994) 2024-11-16T05:40:21,554 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:21,559 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a5582e9ac2d6430aa11aca5efdd5dec4_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a5582e9ac2d6430aa11aca5efdd5dec4_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:21,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c67dc9da5b594b34bcc4ad415f8beacd, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:21,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c67dc9da5b594b34bcc4ad415f8beacd is 175, key is test_row_0/A:col10/1731735620398/Put/seqid=0 2024-11-16T05:40:21,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742037_1213 (size=39949) 2024-11-16T05:40:21,567 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c67dc9da5b594b34bcc4ad415f8beacd 2024-11-16T05:40:21,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735681566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735681569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735681572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735681574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/e0b342fe49d4430790b9b03939563cef is 50, key is test_row_0/B:col10/1731735620398/Put/seqid=0 2024-11-16T05:40:21,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742038_1214 (size=12301) 2024-11-16T05:40:21,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/e0b342fe49d4430790b9b03939563cef 2024-11-16T05:40:21,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/48cf9404a3c448f6b629aad6d8c2bacc is 50, key is test_row_0/C:col10/1731735620398/Put/seqid=0 2024-11-16T05:40:21,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742039_1215 (size=12301) 2024-11-16T05:40:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735681678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735681688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735681688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735681688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735681888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735681898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735681898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:21,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735681898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/48cf9404a3c448f6b629aad6d8c2bacc 2024-11-16T05:40:22,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c67dc9da5b594b34bcc4ad415f8beacd as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c67dc9da5b594b34bcc4ad415f8beacd 2024-11-16T05:40:22,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c67dc9da5b594b34bcc4ad415f8beacd, entries=200, sequenceid=293, filesize=39.0 K 2024-11-16T05:40:22,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/e0b342fe49d4430790b9b03939563cef as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0b342fe49d4430790b9b03939563cef 2024-11-16T05:40:22,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0b342fe49d4430790b9b03939563cef, entries=150, sequenceid=293, filesize=12.0 K 2024-11-16T05:40:22,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/48cf9404a3c448f6b629aad6d8c2bacc as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/48cf9404a3c448f6b629aad6d8c2bacc 2024-11-16T05:40:22,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/48cf9404a3c448f6b629aad6d8c2bacc, entries=150, sequenceid=293, filesize=12.0 K 2024-11-16T05:40:22,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 82b1d437a2b8fce4a136c21be6257221 in 516ms, sequenceid=293, compaction requested=true 2024-11-16T05:40:22,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:22,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:22,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:22,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:22,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:40:22,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:22,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-16T05:40:22,056 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:22,056 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:22,057 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:22,058 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/A is initiating minor compaction (all files) 2024-11-16T05:40:22,058 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/A in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:22,058 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:22,058 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/C is initiating minor compaction (all files) 2024-11-16T05:40:22,058 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/C in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:22,058 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/38746f96170449e2861b398bb7d131dc, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c978a4c093634ea6b832adc7544738c7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c67dc9da5b594b34bcc4ad415f8beacd] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=100.5 K 2024-11-16T05:40:22,058 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:22,058 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5858a96295ca41248d2925bb44179650, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/27739c1bca9d4fd6a0799cd9315ab92f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/48cf9404a3c448f6b629aad6d8c2bacc] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=36.5 K 2024-11-16T05:40:22,058 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/38746f96170449e2861b398bb7d131dc, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c978a4c093634ea6b832adc7544738c7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c67dc9da5b594b34bcc4ad415f8beacd] 2024-11-16T05:40:22,058 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38746f96170449e2861b398bb7d131dc, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731735618063 2024-11-16T05:40:22,058 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5858a96295ca41248d2925bb44179650, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731735618063 2024-11-16T05:40:22,059 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 27739c1bca9d4fd6a0799cd9315ab92f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731735619214 2024-11-16T05:40:22,059 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c978a4c093634ea6b832adc7544738c7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731735619214 2024-11-16T05:40:22,059 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 48cf9404a3c448f6b629aad6d8c2bacc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731735620398 2024-11-16T05:40:22,059 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c67dc9da5b594b34bcc4ad415f8beacd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731735620398 2024-11-16T05:40:22,071 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:22,082 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116a536a85bfa1d450cbd6b87cc632491c7_82b1d437a2b8fce4a136c21be6257221 store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:22,083 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#C#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:22,084 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/e67fba14620f4143ad68b3d9a7388edd is 50, key is test_row_0/C:col10/1731735620398/Put/seqid=0 2024-11-16T05:40:22,084 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116a536a85bfa1d450cbd6b87cc632491c7_82b1d437a2b8fce4a136c21be6257221, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:22,084 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a536a85bfa1d450cbd6b87cc632491c7_82b1d437a2b8fce4a136c21be6257221 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:22,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742040_1216 (size=4469) 2024-11-16T05:40:22,098 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#A#compaction#180 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:22,098 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/b92662bc9ec94246b64af6f1bbd68f1b is 175, key is test_row_0/A:col10/1731735620398/Put/seqid=0 2024-11-16T05:40:22,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742042_1218 (size=31937) 2024-11-16T05:40:22,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742041_1217 (size=12983) 2024-11-16T05:40:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:22,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T05:40:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:22,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165db83a7873444c04a8de5f63dc2e2af5_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735621571/Put/seqid=0 2024-11-16T05:40:22,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742043_1219 (size=14994) 2024-11-16T05:40:22,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735682226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735682229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735682231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735682231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735682338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735682348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735682348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735682348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,513 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/e67fba14620f4143ad68b3d9a7388edd as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e67fba14620f4143ad68b3d9a7388edd 2024-11-16T05:40:22,514 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/b92662bc9ec94246b64af6f1bbd68f1b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b92662bc9ec94246b64af6f1bbd68f1b 2024-11-16T05:40:22,519 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/C of 82b1d437a2b8fce4a136c21be6257221 into e67fba14620f4143ad68b3d9a7388edd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:22,519 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:22,519 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/C, priority=13, startTime=1731735622055; duration=0sec 2024-11-16T05:40:22,519 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:22,519 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:C 2024-11-16T05:40:22,519 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:22,520 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/A of 82b1d437a2b8fce4a136c21be6257221 into b92662bc9ec94246b64af6f1bbd68f1b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:22,521 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:22,521 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/A, priority=13, startTime=1731735622055; duration=0sec 2024-11-16T05:40:22,521 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:22,521 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:A 2024-11-16T05:40:22,522 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:22,522 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/B is initiating minor compaction (all files) 2024-11-16T05:40:22,522 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/B in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:22,522 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/400ca4f626a243b19c0bb19dab7ee344, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/aacd00aea3874767a1fbef280c589c8a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0b342fe49d4430790b9b03939563cef] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=36.5 K 2024-11-16T05:40:22,522 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 400ca4f626a243b19c0bb19dab7ee344, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731735618063 2024-11-16T05:40:22,525 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting aacd00aea3874767a1fbef280c589c8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731735619214 2024-11-16T05:40:22,526 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e0b342fe49d4430790b9b03939563cef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731735620398 2024-11-16T05:40:22,534 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#B#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:22,535 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/8c75f91bed3746f3b5c218fc79a298f7 is 50, key is test_row_0/B:col10/1731735620398/Put/seqid=0 2024-11-16T05:40:22,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735682548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742044_1220 (size=12983) 2024-11-16T05:40:22,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735682559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735682559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735682559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,572 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/8c75f91bed3746f3b5c218fc79a298f7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/8c75f91bed3746f3b5c218fc79a298f7 2024-11-16T05:40:22,582 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/B of 82b1d437a2b8fce4a136c21be6257221 into 8c75f91bed3746f3b5c218fc79a298f7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:22,582 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:22,582 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/B, priority=13, startTime=1731735622055; duration=0sec 2024-11-16T05:40:22,582 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:22,582 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:B 2024-11-16T05:40:22,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:22,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165db83a7873444c04a8de5f63dc2e2af5_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165db83a7873444c04a8de5f63dc2e2af5_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:22,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c711ef051c024172abc69bca53cc8ea0, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:22,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c711ef051c024172abc69bca53cc8ea0 is 175, key is test_row_0/A:col10/1731735621571/Put/seqid=0 2024-11-16T05:40:22,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742045_1221 (size=39949) 2024-11-16T05:40:22,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c711ef051c024172abc69bca53cc8ea0 2024-11-16T05:40:22,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/16eb0bdae9f24c3c839e87ec2d5ccc8c is 50, key is test_row_0/B:col10/1731735621571/Put/seqid=0 2024-11-16T05:40:22,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742046_1222 (size=12301) 2024-11-16T05:40:22,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/16eb0bdae9f24c3c839e87ec2d5ccc8c 2024-11-16T05:40:22,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/e71aea2cf98d4121b0286d25f28a5002 is 50, key is test_row_0/C:col10/1731735621571/Put/seqid=0 2024-11-16T05:40:22,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742047_1223 (size=12301) 2024-11-16T05:40:22,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/e71aea2cf98d4121b0286d25f28a5002 2024-11-16T05:40:22,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/c711ef051c024172abc69bca53cc8ea0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c711ef051c024172abc69bca53cc8ea0 2024-11-16T05:40:22,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c711ef051c024172abc69bca53cc8ea0, entries=200, sequenceid=313, filesize=39.0 K 2024-11-16T05:40:22,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/16eb0bdae9f24c3c839e87ec2d5ccc8c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16eb0bdae9f24c3c839e87ec2d5ccc8c 2024-11-16T05:40:22,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16eb0bdae9f24c3c839e87ec2d5ccc8c, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T05:40:22,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/e71aea2cf98d4121b0286d25f28a5002 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e71aea2cf98d4121b0286d25f28a5002 2024-11-16T05:40:22,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e71aea2cf98d4121b0286d25f28a5002, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T05:40:22,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 82b1d437a2b8fce4a136c21be6257221 in 594ms, sequenceid=313, compaction requested=false 2024-11-16T05:40:22,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:22,864 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:22,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T05:40:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:22,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fd557a006ad84032b3a220038cc256a3_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735622225/Put/seqid=0 2024-11-16T05:40:22,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742048_1224 (size=14994) 2024-11-16T05:40:22,911 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:22,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735682906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735682907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735682909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735682911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fd557a006ad84032b3a220038cc256a3_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fd557a006ad84032b3a220038cc256a3_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:22,922 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/2587b259ee4e42658dbf1d79abe9198c, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:22,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/2587b259ee4e42658dbf1d79abe9198c is 175, key is test_row_0/A:col10/1731735622225/Put/seqid=0 2024-11-16T05:40:22,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742049_1225 (size=39949) 2024-11-16T05:40:23,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735683018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735683029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735683029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735683029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735683228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735683239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735683239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735683239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,327 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/2587b259ee4e42658dbf1d79abe9198c 2024-11-16T05:40:23,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/770aaa217f2f4659ade0c7e0e9a55bef is 50, key is test_row_0/B:col10/1731735622225/Put/seqid=0 2024-11-16T05:40:23,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742050_1226 (size=12301) 2024-11-16T05:40:23,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/770aaa217f2f4659ade0c7e0e9a55bef 2024-11-16T05:40:23,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/1f3cc7208ace450a85c15cbd573499c4 is 50, key is test_row_0/C:col10/1731735622225/Put/seqid=0 2024-11-16T05:40:23,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742051_1227 (size=12301) 2024-11-16T05:40:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T05:40:23,538 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:23,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-16T05:40:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T05:40:23,540 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:23,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735683538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,542 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:23,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:23,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735683548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735683548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735683549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T05:40:23,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T05:40:23,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:23,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:23,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:23,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:23,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:23,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/1f3cc7208ace450a85c15cbd573499c4 2024-11-16T05:40:23,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/2587b259ee4e42658dbf1d79abe9198c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/2587b259ee4e42658dbf1d79abe9198c 2024-11-16T05:40:23,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/2587b259ee4e42658dbf1d79abe9198c, entries=200, sequenceid=333, filesize=39.0 K 2024-11-16T05:40:23,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/770aaa217f2f4659ade0c7e0e9a55bef as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/770aaa217f2f4659ade0c7e0e9a55bef 2024-11-16T05:40:23,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/770aaa217f2f4659ade0c7e0e9a55bef, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T05:40:23,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/1f3cc7208ace450a85c15cbd573499c4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/1f3cc7208ace450a85c15cbd573499c4 2024-11-16T05:40:23,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/1f3cc7208ace450a85c15cbd573499c4, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T05:40:23,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 82b1d437a2b8fce4a136c21be6257221 in 927ms, sequenceid=333, compaction requested=true 2024-11-16T05:40:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:23,792 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:40:23,792 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:23,793 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:23,793 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:23,793 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/B is initiating minor compaction (all files) 2024-11-16T05:40:23,793 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/A is initiating minor compaction (all files) 2024-11-16T05:40:23,793 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/B in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:23,793 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/A in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:23,793 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b92662bc9ec94246b64af6f1bbd68f1b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c711ef051c024172abc69bca53cc8ea0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/2587b259ee4e42658dbf1d79abe9198c] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=109.2 K 2024-11-16T05:40:23,793 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/8c75f91bed3746f3b5c218fc79a298f7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16eb0bdae9f24c3c839e87ec2d5ccc8c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/770aaa217f2f4659ade0c7e0e9a55bef] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=36.7 K 2024-11-16T05:40:23,793 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:23,793 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b92662bc9ec94246b64af6f1bbd68f1b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c711ef051c024172abc69bca53cc8ea0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/2587b259ee4e42658dbf1d79abe9198c] 2024-11-16T05:40:23,794 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c75f91bed3746f3b5c218fc79a298f7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731735620398 2024-11-16T05:40:23,794 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b92662bc9ec94246b64af6f1bbd68f1b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731735620398 2024-11-16T05:40:23,794 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 16eb0bdae9f24c3c839e87ec2d5ccc8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731735621570 2024-11-16T05:40:23,795 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c711ef051c024172abc69bca53cc8ea0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731735621562 2024-11-16T05:40:23,795 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 770aaa217f2f4659ade0c7e0e9a55bef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731735622225 2024-11-16T05:40:23,795 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2587b259ee4e42658dbf1d79abe9198c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731735622225 2024-11-16T05:40:23,804 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:23,807 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111645c8f53b452b40bd985563cae50d7221_82b1d437a2b8fce4a136c21be6257221 store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:23,807 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#B#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:23,808 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/777a4accdf61451d9fafb6f239a49d73 is 50, key is test_row_0/B:col10/1731735622225/Put/seqid=0 2024-11-16T05:40:23,810 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111645c8f53b452b40bd985563cae50d7221_82b1d437a2b8fce4a136c21be6257221, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:23,810 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111645c8f53b452b40bd985563cae50d7221_82b1d437a2b8fce4a136c21be6257221 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:23,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742052_1228 (size=13085) 2024-11-16T05:40:23,838 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/777a4accdf61451d9fafb6f239a49d73 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/777a4accdf61451d9fafb6f239a49d73 2024-11-16T05:40:23,844 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/B of 82b1d437a2b8fce4a136c21be6257221 into 777a4accdf61451d9fafb6f239a49d73(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:23,844 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:23,844 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/B, priority=13, startTime=1731735623792; duration=0sec 2024-11-16T05:40:23,844 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:23,844 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:B 2024-11-16T05:40:23,844 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:23,846 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:23,846 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/C is initiating minor compaction (all files) 2024-11-16T05:40:23,846 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/C in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:23,846 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e67fba14620f4143ad68b3d9a7388edd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e71aea2cf98d4121b0286d25f28a5002, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/1f3cc7208ace450a85c15cbd573499c4] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=36.7 K 2024-11-16T05:40:23,846 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e67fba14620f4143ad68b3d9a7388edd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731735620398 2024-11-16T05:40:23,847 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e71aea2cf98d4121b0286d25f28a5002, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731735621570 2024-11-16T05:40:23,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T05:40:23,847 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3cc7208ace450a85c15cbd573499c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731735622225 2024-11-16T05:40:23,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:23,848 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T05:40:23,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742053_1229 (size=4469) 2024-11-16T05:40:23,851 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#A#compaction#189 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:23,851 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/42677b8eb53542cba1a89cd1c24404d1 is 175, key is test_row_0/A:col10/1731735622225/Put/seqid=0 2024-11-16T05:40:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T05:40:23,865 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#C#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:23,865 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/ed5e0fb9d8504466bdee5b38cabf0981 is 50, key is test_row_0/C:col10/1731735622225/Put/seqid=0 2024-11-16T05:40:23,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742054_1230 (size=32039) 2024-11-16T05:40:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d70227feee774e3d828c237f672ee244_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735622909/Put/seqid=0 2024-11-16T05:40:23,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742055_1231 (size=13085) 2024-11-16T05:40:23,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742056_1232 (size=12454) 2024-11-16T05:40:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:23,917 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/ed5e0fb9d8504466bdee5b38cabf0981 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/ed5e0fb9d8504466bdee5b38cabf0981 2024-11-16T05:40:23,921 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d70227feee774e3d828c237f672ee244_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d70227feee774e3d828c237f672ee244_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:23,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/215b9a20892742c7971327d06143a541, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:23,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/215b9a20892742c7971327d06143a541 is 175, key is test_row_0/A:col10/1731735622909/Put/seqid=0 2024-11-16T05:40:23,927 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/C of 82b1d437a2b8fce4a136c21be6257221 into ed5e0fb9d8504466bdee5b38cabf0981(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:23,927 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:23,927 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/C, priority=13, startTime=1731735623792; duration=0sec 2024-11-16T05:40:23,927 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:23,927 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:C 2024-11-16T05:40:23,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742057_1233 (size=31255) 2024-11-16T05:40:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:24,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:24,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735684116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735684117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735684118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735684118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T05:40:24,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735684228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735684228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735684228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735684228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,290 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/42677b8eb53542cba1a89cd1c24404d1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/42677b8eb53542cba1a89cd1c24404d1 2024-11-16T05:40:24,300 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/A of 82b1d437a2b8fce4a136c21be6257221 into 42677b8eb53542cba1a89cd1c24404d1(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:24,300 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:24,300 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/A, priority=13, startTime=1731735623792; duration=0sec 2024-11-16T05:40:24,300 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:24,300 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:A 2024-11-16T05:40:24,339 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/215b9a20892742c7971327d06143a541 2024-11-16T05:40:24,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/6cabc95961934ad9b8d4ccf5344e4e36 is 50, key is test_row_0/B:col10/1731735622909/Put/seqid=0 2024-11-16T05:40:24,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742058_1234 (size=12301) 2024-11-16T05:40:24,364 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/6cabc95961934ad9b8d4ccf5344e4e36 2024-11-16T05:40:24,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/cc768c28c7f841feb3fff3a815273291 is 50, key is test_row_0/C:col10/1731735622909/Put/seqid=0 2024-11-16T05:40:24,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742059_1235 (size=12301) 2024-11-16T05:40:24,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735684438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735684438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735684439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735684439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T05:40:24,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735684748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735684748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735684748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735684749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:24,777 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/cc768c28c7f841feb3fff3a815273291 2024-11-16T05:40:24,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/215b9a20892742c7971327d06143a541 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/215b9a20892742c7971327d06143a541 2024-11-16T05:40:24,796 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/215b9a20892742c7971327d06143a541, entries=150, sequenceid=353, filesize=30.5 K 2024-11-16T05:40:24,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/6cabc95961934ad9b8d4ccf5344e4e36 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/6cabc95961934ad9b8d4ccf5344e4e36 2024-11-16T05:40:24,811 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/6cabc95961934ad9b8d4ccf5344e4e36, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T05:40:24,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/cc768c28c7f841feb3fff3a815273291 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/cc768c28c7f841feb3fff3a815273291 2024-11-16T05:40:24,818 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/cc768c28c7f841feb3fff3a815273291, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T05:40:24,819 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 82b1d437a2b8fce4a136c21be6257221 in 972ms, sequenceid=353, compaction requested=false 2024-11-16T05:40:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-16T05:40:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-16T05:40:24,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-16T05:40:24,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2780 sec 2024-11-16T05:40:24,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.2830 sec 2024-11-16T05:40:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:25,275 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:25,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T05:40:25,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b1507727179b4e0eac6b73161e27fb49_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735625258/Put/seqid=0 2024-11-16T05:40:25,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742060_1236 (size=12454) 2024-11-16T05:40:25,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735685303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735685306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735685307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735685309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735685418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735685418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735685418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735685418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735685628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735685628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735685628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735685629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T05:40:25,698 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:25,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-16T05:40:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:40:25,700 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:25,701 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:25,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:25,701 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:25,705 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b1507727179b4e0eac6b73161e27fb49_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b1507727179b4e0eac6b73161e27fb49_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:25,706 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/e2c8d762dd9c408392e01b96e50089ba, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:25,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/e2c8d762dd9c408392e01b96e50089ba is 175, key is test_row_0/A:col10/1731735625258/Put/seqid=0 2024-11-16T05:40:25,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742061_1237 (size=31255) 2024-11-16T05:40:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:40:25,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:40:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:25,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:25,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735685938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735685938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735685939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:25,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735685939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:25,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:25,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:26,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:40:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:40:26,111 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/e2c8d762dd9c408392e01b96e50089ba 2024-11-16T05:40:26,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/424286bd785446349f86b3dfd56cdc39 is 50, key is test_row_0/B:col10/1731735625258/Put/seqid=0 2024-11-16T05:40:26,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742062_1238 (size=12301) 2024-11-16T05:40:26,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:40:26,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:26,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:40:26,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:26,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:40:26,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735686449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:26,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735686449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:26,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735686449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:26,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735686449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:26,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:40:26,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:26,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/424286bd785446349f86b3dfd56cdc39 2024-11-16T05:40:26,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/de7fd635a12149528b7c4c65d0a43ab8 is 50, key is test_row_0/C:col10/1731735625258/Put/seqid=0 2024-11-16T05:40:26,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742063_1239 (size=12301) 2024-11-16T05:40:26,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:40:26,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:26,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:40:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:40:26,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:40:26,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:26,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:26,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/de7fd635a12149528b7c4c65d0a43ab8 2024-11-16T05:40:26,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/e2c8d762dd9c408392e01b96e50089ba as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/e2c8d762dd9c408392e01b96e50089ba 2024-11-16T05:40:26,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/e2c8d762dd9c408392e01b96e50089ba, entries=150, sequenceid=375, filesize=30.5 K 2024-11-16T05:40:26,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/424286bd785446349f86b3dfd56cdc39 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/424286bd785446349f86b3dfd56cdc39 2024-11-16T05:40:26,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/424286bd785446349f86b3dfd56cdc39, entries=150, sequenceid=375, filesize=12.0 K 2024-11-16T05:40:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/de7fd635a12149528b7c4c65d0a43ab8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/de7fd635a12149528b7c4c65d0a43ab8 2024-11-16T05:40:26,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/de7fd635a12149528b7c4c65d0a43ab8, entries=150, sequenceid=375, filesize=12.0 K 2024-11-16T05:40:26,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 82b1d437a2b8fce4a136c21be6257221 in 1680ms, sequenceid=375, compaction requested=true 2024-11-16T05:40:26,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:26,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:26,955 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:26,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:26,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:26,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:26,955 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:26,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:26,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:26,957 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:26,957 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:26,957 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/B is initiating minor compaction (all files) 2024-11-16T05:40:26,957 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/A is initiating minor compaction (all files) 2024-11-16T05:40:26,957 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/B in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,957 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/A in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,957 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/42677b8eb53542cba1a89cd1c24404d1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/215b9a20892742c7971327d06143a541, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/e2c8d762dd9c408392e01b96e50089ba] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=92.3 K 2024-11-16T05:40:26,957 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/777a4accdf61451d9fafb6f239a49d73, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/6cabc95961934ad9b8d4ccf5344e4e36, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/424286bd785446349f86b3dfd56cdc39] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=36.8 K 2024-11-16T05:40:26,957 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:26,957 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/42677b8eb53542cba1a89cd1c24404d1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/215b9a20892742c7971327d06143a541, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/e2c8d762dd9c408392e01b96e50089ba] 2024-11-16T05:40:26,957 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 777a4accdf61451d9fafb6f239a49d73, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731735622225 2024-11-16T05:40:26,958 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42677b8eb53542cba1a89cd1c24404d1, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731735622225 2024-11-16T05:40:26,958 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cabc95961934ad9b8d4ccf5344e4e36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731735622903 2024-11-16T05:40:26,958 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 215b9a20892742c7971327d06143a541, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731735622903 2024-11-16T05:40:26,959 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 424286bd785446349f86b3dfd56cdc39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731735625258 2024-11-16T05:40:26,959 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2c8d762dd9c408392e01b96e50089ba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731735625258 2024-11-16T05:40:26,966 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#B#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:26,967 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/352d9b600d8d44bfa7805b3f814fa131 is 50, key is test_row_0/B:col10/1731735625258/Put/seqid=0 2024-11-16T05:40:26,972 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:26,975 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116f5b793635839488ca66f69f802624347_82b1d437a2b8fce4a136c21be6257221 store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:26,977 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116f5b793635839488ca66f69f802624347_82b1d437a2b8fce4a136c21be6257221, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:26,977 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f5b793635839488ca66f69f802624347_82b1d437a2b8fce4a136c21be6257221 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:26,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742065_1241 (size=4469) 2024-11-16T05:40:26,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742064_1240 (size=13187) 2024-11-16T05:40:26,994 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/352d9b600d8d44bfa7805b3f814fa131 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/352d9b600d8d44bfa7805b3f814fa131 2024-11-16T05:40:26,998 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/B of 82b1d437a2b8fce4a136c21be6257221 into 352d9b600d8d44bfa7805b3f814fa131(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:26,998 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:26,998 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/B, priority=13, startTime=1731735626955; duration=0sec 2024-11-16T05:40:26,998 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:26,998 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:B 2024-11-16T05:40:26,998 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:26,999 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:27,000 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 82b1d437a2b8fce4a136c21be6257221/C is initiating minor compaction (all files) 2024-11-16T05:40:27,000 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b1d437a2b8fce4a136c21be6257221/C in TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:27,000 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/ed5e0fb9d8504466bdee5b38cabf0981, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/cc768c28c7f841feb3fff3a815273291, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/de7fd635a12149528b7c4c65d0a43ab8] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp, totalSize=36.8 K 2024-11-16T05:40:27,000 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ed5e0fb9d8504466bdee5b38cabf0981, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731735622225 2024-11-16T05:40:27,000 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting cc768c28c7f841feb3fff3a815273291, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731735622903 2024-11-16T05:40:27,001 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting de7fd635a12149528b7c4c65d0a43ab8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731735625258 2024-11-16T05:40:27,007 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#C#compaction#200 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:27,007 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/c461620d3bd44413a2e9778e338ab047 is 50, key is test_row_0/C:col10/1731735625258/Put/seqid=0 2024-11-16T05:40:27,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742066_1242 (size=13187) 2024-11-16T05:40:27,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:40:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:27,077 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T05:40:27,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111655bd84258df849c595ba0f40bd98528d_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735625308/Put/seqid=0 2024-11-16T05:40:27,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742067_1243 (size=12454) 2024-11-16T05:40:27,387 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b1d437a2b8fce4a136c21be6257221#A#compaction#199 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:27,387 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/1201fa175e04478d9302f1b3b5a46692 is 175, key is test_row_0/A:col10/1731735625258/Put/seqid=0 2024-11-16T05:40:27,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742068_1244 (size=32141) 2024-11-16T05:40:27,400 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/1201fa175e04478d9302f1b3b5a46692 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/1201fa175e04478d9302f1b3b5a46692 2024-11-16T05:40:27,406 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/A of 82b1d437a2b8fce4a136c21be6257221 into 1201fa175e04478d9302f1b3b5a46692(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:27,406 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:27,406 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/A, priority=13, startTime=1731735626955; duration=0sec 2024-11-16T05:40:27,406 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:27,406 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:A 2024-11-16T05:40:27,423 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/c461620d3bd44413a2e9778e338ab047 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/c461620d3bd44413a2e9778e338ab047 2024-11-16T05:40:27,431 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b1d437a2b8fce4a136c21be6257221/C of 82b1d437a2b8fce4a136c21be6257221 into c461620d3bd44413a2e9778e338ab047(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:27,431 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:27,431 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., storeName=82b1d437a2b8fce4a136c21be6257221/C, priority=13, startTime=1731735626955; duration=0sec 2024-11-16T05:40:27,431 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:27,431 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:C 2024-11-16T05:40:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:27,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. as already flushing 2024-11-16T05:40:27,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735687491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735687492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735687493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735687496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,519 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111655bd84258df849c595ba0f40bd98528d_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111655bd84258df849c595ba0f40bd98528d_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/d036b6d372a44f158c799c5343f267db, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/d036b6d372a44f158c799c5343f267db is 175, key is test_row_0/A:col10/1731735625308/Put/seqid=0 2024-11-16T05:40:27,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742069_1245 (size=31255) 2024-11-16T05:40:27,581 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-16T05:40:27,582 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:27,582 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:27,582 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-16T05:40:27,582 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:27,582 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:27,582 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:27,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:27,582 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:27,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:27,583 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-16T05:40:27,583 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:27,583 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:27,583 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:27,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:27,586 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-16T05:40:27,586 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:27,586 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:27,586 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:27,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:27,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735687598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735687608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735687608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735687608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735687808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735687818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735687818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735687818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:40:27,926 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/d036b6d372a44f158c799c5343f267db 2024-11-16T05:40:27,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/5e994e86c1a8442a932b0ef84addb369 is 50, key is test_row_0/B:col10/1731735625308/Put/seqid=0 2024-11-16T05:40:27,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742070_1246 (size=12301) 2024-11-16T05:40:28,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735688118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:28,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:28,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735688129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735688129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:28,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735688129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:28,340 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/5e994e86c1a8442a932b0ef84addb369 2024-11-16T05:40:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/9a00464b814744849c119317df8f4ce7 is 50, key is test_row_0/C:col10/1731735625308/Put/seqid=0 2024-11-16T05:40:28,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742071_1247 (size=12301) 2024-11-16T05:40:28,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55356 deadline: 1731735688628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:28,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:28,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55404 deadline: 1731735688638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:28,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55358 deadline: 1731735688638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55412 deadline: 1731735688638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:40:28,756 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/9a00464b814744849c119317df8f4ce7 2024-11-16T05:40:28,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/d036b6d372a44f158c799c5343f267db as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/d036b6d372a44f158c799c5343f267db 2024-11-16T05:40:28,765 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/d036b6d372a44f158c799c5343f267db, entries=150, sequenceid=392, filesize=30.5 K 2024-11-16T05:40:28,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/5e994e86c1a8442a932b0ef84addb369 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/5e994e86c1a8442a932b0ef84addb369 2024-11-16T05:40:28,770 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/5e994e86c1a8442a932b0ef84addb369, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T05:40:28,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/9a00464b814744849c119317df8f4ce7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9a00464b814744849c119317df8f4ce7 2024-11-16T05:40:28,776 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9a00464b814744849c119317df8f4ce7, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T05:40:28,778 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 82b1d437a2b8fce4a136c21be6257221 in 1701ms, sequenceid=392, compaction requested=false 2024-11-16T05:40:28,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:28,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:28,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-16T05:40:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-16T05:40:28,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-16T05:40:28,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0780 sec 2024-11-16T05:40:28,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 3.0830 sec 2024-11-16T05:40:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:29,631 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:29,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T05:40:29,631 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-16T05:40:29,631 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:29,631 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,631 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:29,640 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-16T05:40:29,640 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:29,640 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dc94bdb702824881a4c7f3d7b710cefb_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_0/A:col10/1731735627484/Put/seqid=0 2024-11-16T05:40:29,640 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:29,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742072_1248 (size=12454) 2024-11-16T05:40:29,650 INFO [Thread-679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-679. 2024-11-16T05:40:29,650 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-16T05:40:29,650 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:29,650 DEBUG [Thread-679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:29,650 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,650 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,650 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-16T05:40:29,650 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,650 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:29,650 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:29,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:29,650 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,650 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:40:29,859 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3118 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3107 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1381 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4143 rows 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1391 2024-11-16T05:40:29,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4168 rows 2024-11-16T05:40:29,859 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:40:29,859 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:40:29,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:29,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:29,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:40:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:40:29,864 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735629863"}]},"ts":"1731735629863"} 2024-11-16T05:40:29,865 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:40:29,865 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:40:29,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:40:29,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, UNASSIGN}] 2024-11-16T05:40:29,868 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, UNASSIGN 2024-11-16T05:40:29,869 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=82b1d437a2b8fce4a136c21be6257221, regionState=CLOSING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:29,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, UNASSIGN because future has completed 2024-11-16T05:40:29,871 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:40:29,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:40:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:40:30,023 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:30,023 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:40:30,024 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 82b1d437a2b8fce4a136c21be6257221, disabling compactions & flushes 2024-11-16T05:40:30,024 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:30,045 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:30,051 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dc94bdb702824881a4c7f3d7b710cefb_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dc94bdb702824881a4c7f3d7b710cefb_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:30,052 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/bee9efd1326f4b2bb2fdf37b9aa1dc77, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:30,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/bee9efd1326f4b2bb2fdf37b9aa1dc77 is 175, key is test_row_0/A:col10/1731735627484/Put/seqid=0 2024-11-16T05:40:30,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742073_1249 (size=31255) 2024-11-16T05:40:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:40:30,460 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=415, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/bee9efd1326f4b2bb2fdf37b9aa1dc77 2024-11-16T05:40:30,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/7dbad459bfa047978850434b09acac21 is 50, key is test_row_0/B:col10/1731735627484/Put/seqid=0 2024-11-16T05:40:30,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742074_1250 (size=12301) 2024-11-16T05:40:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:40:30,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/7dbad459bfa047978850434b09acac21 2024-11-16T05:40:30,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/e3e33b5d306d4cfe8049d9177a821b58 is 50, key is test_row_0/C:col10/1731735627484/Put/seqid=0 2024-11-16T05:40:30,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742075_1251 (size=12301) 2024-11-16T05:40:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:40:31,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/e3e33b5d306d4cfe8049d9177a821b58 2024-11-16T05:40:31,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/bee9efd1326f4b2bb2fdf37b9aa1dc77 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/bee9efd1326f4b2bb2fdf37b9aa1dc77 2024-11-16T05:40:31,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/bee9efd1326f4b2bb2fdf37b9aa1dc77, entries=150, sequenceid=415, filesize=30.5 K 2024-11-16T05:40:31,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/7dbad459bfa047978850434b09acac21 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/7dbad459bfa047978850434b09acac21 2024-11-16T05:40:31,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/7dbad459bfa047978850434b09acac21, entries=150, sequenceid=415, filesize=12.0 K 2024-11-16T05:40:31,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/e3e33b5d306d4cfe8049d9177a821b58 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e3e33b5d306d4cfe8049d9177a821b58 2024-11-16T05:40:31,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e3e33b5d306d4cfe8049d9177a821b58, entries=150, sequenceid=415, filesize=12.0 K 2024-11-16T05:40:31,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=26.84 KB/27480 for 82b1d437a2b8fce4a136c21be6257221 in 1698ms, sequenceid=415, compaction requested=true 2024-11-16T05:40:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b1d437a2b8fce4a136c21be6257221: 2024-11-16T05:40:31,329 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:31,329 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:31,329 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. after waiting 0 ms 2024-11-16T05:40:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:31,329 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b1d437a2b8fce4a136c21be6257221:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:31,329 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. because compaction request was cancelled 2024-11-16T05:40:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:31,329 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:A 2024-11-16T05:40:31,329 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. because compaction request was cancelled 2024-11-16T05:40:31,329 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 82b1d437a2b8fce4a136c21be6257221 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-16T05:40:31,329 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. because compaction request was cancelled 2024-11-16T05:40:31,329 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:B 2024-11-16T05:40:31,329 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b1d437a2b8fce4a136c21be6257221:C 2024-11-16T05:40:31,336 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dafa67dd84ae4267811d380f65fedb5d_82b1d437a2b8fce4a136c21be6257221 is 50, key is test_row_1/A:col10/1731735629638/Put/seqid=0 2024-11-16T05:40:31,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742076_1252 (size=9914) 2024-11-16T05:40:31,741 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:31,746 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dafa67dd84ae4267811d380f65fedb5d_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dafa67dd84ae4267811d380f65fedb5d_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:31,747 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3c9baaf74f4e476a857f33062fa01e86, store: [table=TestAcidGuarantees family=A region=82b1d437a2b8fce4a136c21be6257221] 2024-11-16T05:40:31,748 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3c9baaf74f4e476a857f33062fa01e86 is 175, key is test_row_1/A:col10/1731735629638/Put/seqid=0 2024-11-16T05:40:31,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742077_1253 (size=22561) 2024-11-16T05:40:31,754 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3c9baaf74f4e476a857f33062fa01e86 2024-11-16T05:40:31,760 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/95c7ecdb7dbb424d95d76215b1a870bc is 50, key is test_row_1/B:col10/1731735629638/Put/seqid=0 2024-11-16T05:40:31,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742078_1254 (size=9857) 2024-11-16T05:40:32,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:40:32,165 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/95c7ecdb7dbb424d95d76215b1a870bc 2024-11-16T05:40:32,172 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/05e17ee5334c4714b064bb5c7146e729 is 50, key is test_row_1/C:col10/1731735629638/Put/seqid=0 2024-11-16T05:40:32,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742079_1255 (size=9857) 2024-11-16T05:40:32,577 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/05e17ee5334c4714b064bb5c7146e729 2024-11-16T05:40:32,588 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/A/3c9baaf74f4e476a857f33062fa01e86 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3c9baaf74f4e476a857f33062fa01e86 2024-11-16T05:40:32,593 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3c9baaf74f4e476a857f33062fa01e86, entries=100, sequenceid=422, filesize=22.0 K 2024-11-16T05:40:32,594 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/B/95c7ecdb7dbb424d95d76215b1a870bc as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/95c7ecdb7dbb424d95d76215b1a870bc 2024-11-16T05:40:32,599 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/95c7ecdb7dbb424d95d76215b1a870bc, entries=100, sequenceid=422, filesize=9.6 K 2024-11-16T05:40:32,600 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/.tmp/C/05e17ee5334c4714b064bb5c7146e729 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/05e17ee5334c4714b064bb5c7146e729 2024-11-16T05:40:32,603 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/05e17ee5334c4714b064bb5c7146e729, entries=100, sequenceid=422, filesize=9.6 K 2024-11-16T05:40:32,604 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 82b1d437a2b8fce4a136c21be6257221 in 1275ms, sequenceid=422, compaction requested=true 2024-11-16T05:40:32,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/15dc7b60cfea4b159c98f557dcd58277, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3232ca2bbd1e42c8b23c0904e9feea73, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a40e057451f04fab85062ead9f32a49c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/6cbac66200c44e7dbeb7655f58a438eb, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7af93718e1b1483c987dab59b8c89b13, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/51df5eb87daf476e86f578580bc472c8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0276652e38a14b42816d7c29518a2444, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7ade7b77003f4452a6ea9ec1cd81ce7d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0a43e4e2a6df4a0ba241040d39b56e54, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/765a2a1d52a342eeaf245204f0d74e15, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/9109ef51b2b24d0ca2ae94d878acb3d8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/d3f52d8cd3b7479fa3b586d8e3ff9869, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/27d229003f8d47d0b7d63838a64adb92, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/74195d8290e84110aa818323ba038b35, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a56a28545dd8452fa72cbf98b9acd2a8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b8e3e607d0c54bb79deff32cc6d57c61, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3fcce2754cb8427c9cfd60b8d7bc27cd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/38746f96170449e2861b398bb7d131dc, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c978a4c093634ea6b832adc7544738c7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c67dc9da5b594b34bcc4ad415f8beacd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b92662bc9ec94246b64af6f1bbd68f1b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c711ef051c024172abc69bca53cc8ea0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/2587b259ee4e42658dbf1d79abe9198c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/42677b8eb53542cba1a89cd1c24404d1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/215b9a20892742c7971327d06143a541, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/e2c8d762dd9c408392e01b96e50089ba] to archive 2024-11-16T05:40:32,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:40:32,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/15dc7b60cfea4b159c98f557dcd58277 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/15dc7b60cfea4b159c98f557dcd58277 2024-11-16T05:40:32,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3232ca2bbd1e42c8b23c0904e9feea73 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3232ca2bbd1e42c8b23c0904e9feea73 2024-11-16T05:40:32,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a40e057451f04fab85062ead9f32a49c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a40e057451f04fab85062ead9f32a49c 2024-11-16T05:40:32,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/6cbac66200c44e7dbeb7655f58a438eb to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/6cbac66200c44e7dbeb7655f58a438eb 2024-11-16T05:40:32,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7af93718e1b1483c987dab59b8c89b13 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7af93718e1b1483c987dab59b8c89b13 2024-11-16T05:40:32,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/51df5eb87daf476e86f578580bc472c8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/51df5eb87daf476e86f578580bc472c8 2024-11-16T05:40:32,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0276652e38a14b42816d7c29518a2444 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0276652e38a14b42816d7c29518a2444 2024-11-16T05:40:32,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7ade7b77003f4452a6ea9ec1cd81ce7d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/7ade7b77003f4452a6ea9ec1cd81ce7d 2024-11-16T05:40:32,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0a43e4e2a6df4a0ba241040d39b56e54 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/0a43e4e2a6df4a0ba241040d39b56e54 2024-11-16T05:40:32,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/765a2a1d52a342eeaf245204f0d74e15 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/765a2a1d52a342eeaf245204f0d74e15 2024-11-16T05:40:32,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/9109ef51b2b24d0ca2ae94d878acb3d8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/9109ef51b2b24d0ca2ae94d878acb3d8 2024-11-16T05:40:32,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/d3f52d8cd3b7479fa3b586d8e3ff9869 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/d3f52d8cd3b7479fa3b586d8e3ff9869 2024-11-16T05:40:32,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/27d229003f8d47d0b7d63838a64adb92 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/27d229003f8d47d0b7d63838a64adb92 2024-11-16T05:40:32,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/74195d8290e84110aa818323ba038b35 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/74195d8290e84110aa818323ba038b35 2024-11-16T05:40:32,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a56a28545dd8452fa72cbf98b9acd2a8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/a56a28545dd8452fa72cbf98b9acd2a8 2024-11-16T05:40:32,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b8e3e607d0c54bb79deff32cc6d57c61 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b8e3e607d0c54bb79deff32cc6d57c61 2024-11-16T05:40:32,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3fcce2754cb8427c9cfd60b8d7bc27cd to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3fcce2754cb8427c9cfd60b8d7bc27cd 2024-11-16T05:40:32,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/38746f96170449e2861b398bb7d131dc to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/38746f96170449e2861b398bb7d131dc 2024-11-16T05:40:32,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c978a4c093634ea6b832adc7544738c7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c978a4c093634ea6b832adc7544738c7 2024-11-16T05:40:32,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c67dc9da5b594b34bcc4ad415f8beacd to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c67dc9da5b594b34bcc4ad415f8beacd 2024-11-16T05:40:32,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b92662bc9ec94246b64af6f1bbd68f1b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/b92662bc9ec94246b64af6f1bbd68f1b 2024-11-16T05:40:32,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c711ef051c024172abc69bca53cc8ea0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/c711ef051c024172abc69bca53cc8ea0 2024-11-16T05:40:32,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/2587b259ee4e42658dbf1d79abe9198c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/2587b259ee4e42658dbf1d79abe9198c 2024-11-16T05:40:32,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/42677b8eb53542cba1a89cd1c24404d1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/42677b8eb53542cba1a89cd1c24404d1 2024-11-16T05:40:32,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/215b9a20892742c7971327d06143a541 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/215b9a20892742c7971327d06143a541 2024-11-16T05:40:32,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/e2c8d762dd9c408392e01b96e50089ba to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/e2c8d762dd9c408392e01b96e50089ba 2024-11-16T05:40:32,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/605797a240684449a029c27ac5354808, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/53b19a4ab29f4a7b819153f36a205f5e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/054cce3227634879bcea95a687bb20b1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/a02e65c0d3d149dc93186cb0f368644f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/5269d7e90a08404eb3a75574c638d9eb, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/b736cea335234d73b169368cea2ec1ad, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/ec1ea3647ece4da1a20da6c77e214b0b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/66406e886cbf4b7ca5ae80b3630917e1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16d3ccd1257a424e92d57e883ae4eb90, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0747da12e824b6aa7f7699c604a767f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/0b7629a77bc54a468f7fa23c09d283ab, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/9463b073e1944319b159bcc095d90c8d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/54865ad5eeb04bc69dc4ae9dd0748e2c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f227b1c455564ad19c04f472b1278655, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/c9349eb043de44d08cfd3a7344fb823c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e34a407f7f2e4ff79ed1b581d2edd9a6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/400ca4f626a243b19c0bb19dab7ee344, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f394a9289ba948eb861895568372e8d2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/aacd00aea3874767a1fbef280c589c8a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/8c75f91bed3746f3b5c218fc79a298f7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0b342fe49d4430790b9b03939563cef, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16eb0bdae9f24c3c839e87ec2d5ccc8c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/777a4accdf61451d9fafb6f239a49d73, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/770aaa217f2f4659ade0c7e0e9a55bef, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/6cabc95961934ad9b8d4ccf5344e4e36, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/424286bd785446349f86b3dfd56cdc39] to archive 2024-11-16T05:40:32,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:40:32,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/605797a240684449a029c27ac5354808 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/605797a240684449a029c27ac5354808 2024-11-16T05:40:32,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/53b19a4ab29f4a7b819153f36a205f5e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/53b19a4ab29f4a7b819153f36a205f5e 2024-11-16T05:40:32,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/054cce3227634879bcea95a687bb20b1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/054cce3227634879bcea95a687bb20b1 2024-11-16T05:40:32,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/a02e65c0d3d149dc93186cb0f368644f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/a02e65c0d3d149dc93186cb0f368644f 2024-11-16T05:40:32,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/5269d7e90a08404eb3a75574c638d9eb to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/5269d7e90a08404eb3a75574c638d9eb 2024-11-16T05:40:32,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/b736cea335234d73b169368cea2ec1ad to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/b736cea335234d73b169368cea2ec1ad 2024-11-16T05:40:32,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/ec1ea3647ece4da1a20da6c77e214b0b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/ec1ea3647ece4da1a20da6c77e214b0b 2024-11-16T05:40:32,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/66406e886cbf4b7ca5ae80b3630917e1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/66406e886cbf4b7ca5ae80b3630917e1 2024-11-16T05:40:32,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16d3ccd1257a424e92d57e883ae4eb90 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16d3ccd1257a424e92d57e883ae4eb90 2024-11-16T05:40:32,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0747da12e824b6aa7f7699c604a767f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0747da12e824b6aa7f7699c604a767f 2024-11-16T05:40:32,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/0b7629a77bc54a468f7fa23c09d283ab to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/0b7629a77bc54a468f7fa23c09d283ab 2024-11-16T05:40:32,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/9463b073e1944319b159bcc095d90c8d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/9463b073e1944319b159bcc095d90c8d 2024-11-16T05:40:32,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/54865ad5eeb04bc69dc4ae9dd0748e2c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/54865ad5eeb04bc69dc4ae9dd0748e2c 2024-11-16T05:40:32,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f227b1c455564ad19c04f472b1278655 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f227b1c455564ad19c04f472b1278655 2024-11-16T05:40:32,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/c9349eb043de44d08cfd3a7344fb823c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/c9349eb043de44d08cfd3a7344fb823c 2024-11-16T05:40:32,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e34a407f7f2e4ff79ed1b581d2edd9a6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e34a407f7f2e4ff79ed1b581d2edd9a6 2024-11-16T05:40:32,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/400ca4f626a243b19c0bb19dab7ee344 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/400ca4f626a243b19c0bb19dab7ee344 2024-11-16T05:40:32,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f394a9289ba948eb861895568372e8d2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/f394a9289ba948eb861895568372e8d2 2024-11-16T05:40:32,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/aacd00aea3874767a1fbef280c589c8a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/aacd00aea3874767a1fbef280c589c8a 2024-11-16T05:40:32,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/8c75f91bed3746f3b5c218fc79a298f7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/8c75f91bed3746f3b5c218fc79a298f7 2024-11-16T05:40:32,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0b342fe49d4430790b9b03939563cef to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/e0b342fe49d4430790b9b03939563cef 2024-11-16T05:40:32,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16eb0bdae9f24c3c839e87ec2d5ccc8c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/16eb0bdae9f24c3c839e87ec2d5ccc8c 2024-11-16T05:40:32,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/777a4accdf61451d9fafb6f239a49d73 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/777a4accdf61451d9fafb6f239a49d73 2024-11-16T05:40:32,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/770aaa217f2f4659ade0c7e0e9a55bef to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/770aaa217f2f4659ade0c7e0e9a55bef 2024-11-16T05:40:32,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/6cabc95961934ad9b8d4ccf5344e4e36 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/6cabc95961934ad9b8d4ccf5344e4e36 2024-11-16T05:40:32,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/424286bd785446349f86b3dfd56cdc39 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/424286bd785446349f86b3dfd56cdc39 2024-11-16T05:40:32,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/d3a1c0fbfc354e9094b5efa972ece3e0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f9c3bb6dec734e99919b4af44e13c3ac, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9e80331dfb6d4065aa811531463efdd2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/2cf152243af94cdc8a77aa7c8b129612, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f978c2da743145ef87884510662616e2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5d6de6f501dc43c79c4f5666446d8cbe, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8a9d372ec1924b59b6bcbcb5db8cdd83, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/847b3562ab554e328248f71cfd2f1bf4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f7393f26c6304c3796c8b80509632890, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/92f9ffada0de4b1fbbed5f706e5e74df, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/01523a38e3234f62950f269f8441ae86, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/dda285d57a6d4aef8a4ce53f63d9e6ed, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8cc52cf81f3c4b56a314b019e2fdc2b4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/7ec35e8782f049f1baf5999656f9f5d2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/652c4c7b4a6349ccbca3d7c36949478a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/89289f095fd545b1a5f89cc516d4acbe, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5858a96295ca41248d2925bb44179650, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9f3b2f785c0540579c36dfe6ef665f29, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/27739c1bca9d4fd6a0799cd9315ab92f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e67fba14620f4143ad68b3d9a7388edd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/48cf9404a3c448f6b629aad6d8c2bacc, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e71aea2cf98d4121b0286d25f28a5002, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/ed5e0fb9d8504466bdee5b38cabf0981, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/1f3cc7208ace450a85c15cbd573499c4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/cc768c28c7f841feb3fff3a815273291, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/de7fd635a12149528b7c4c65d0a43ab8] to archive 2024-11-16T05:40:32,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:40:32,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/d3a1c0fbfc354e9094b5efa972ece3e0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/d3a1c0fbfc354e9094b5efa972ece3e0 2024-11-16T05:40:32,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f9c3bb6dec734e99919b4af44e13c3ac to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f9c3bb6dec734e99919b4af44e13c3ac 2024-11-16T05:40:32,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9e80331dfb6d4065aa811531463efdd2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9e80331dfb6d4065aa811531463efdd2 2024-11-16T05:40:32,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/2cf152243af94cdc8a77aa7c8b129612 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/2cf152243af94cdc8a77aa7c8b129612 2024-11-16T05:40:32,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f978c2da743145ef87884510662616e2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f978c2da743145ef87884510662616e2 2024-11-16T05:40:32,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5d6de6f501dc43c79c4f5666446d8cbe to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5d6de6f501dc43c79c4f5666446d8cbe 2024-11-16T05:40:32,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8a9d372ec1924b59b6bcbcb5db8cdd83 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8a9d372ec1924b59b6bcbcb5db8cdd83 2024-11-16T05:40:32,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/847b3562ab554e328248f71cfd2f1bf4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/847b3562ab554e328248f71cfd2f1bf4 2024-11-16T05:40:32,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f7393f26c6304c3796c8b80509632890 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/f7393f26c6304c3796c8b80509632890 2024-11-16T05:40:32,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/92f9ffada0de4b1fbbed5f706e5e74df to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/92f9ffada0de4b1fbbed5f706e5e74df 2024-11-16T05:40:32,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/01523a38e3234f62950f269f8441ae86 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/01523a38e3234f62950f269f8441ae86 2024-11-16T05:40:32,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/dda285d57a6d4aef8a4ce53f63d9e6ed to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/dda285d57a6d4aef8a4ce53f63d9e6ed 2024-11-16T05:40:32,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8cc52cf81f3c4b56a314b019e2fdc2b4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/8cc52cf81f3c4b56a314b019e2fdc2b4 2024-11-16T05:40:32,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/7ec35e8782f049f1baf5999656f9f5d2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/7ec35e8782f049f1baf5999656f9f5d2 2024-11-16T05:40:32,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/652c4c7b4a6349ccbca3d7c36949478a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/652c4c7b4a6349ccbca3d7c36949478a 2024-11-16T05:40:32,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/89289f095fd545b1a5f89cc516d4acbe to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/89289f095fd545b1a5f89cc516d4acbe 2024-11-16T05:40:32,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5858a96295ca41248d2925bb44179650 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/5858a96295ca41248d2925bb44179650 2024-11-16T05:40:32,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9f3b2f785c0540579c36dfe6ef665f29 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9f3b2f785c0540579c36dfe6ef665f29 2024-11-16T05:40:32,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/27739c1bca9d4fd6a0799cd9315ab92f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/27739c1bca9d4fd6a0799cd9315ab92f 2024-11-16T05:40:32,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e67fba14620f4143ad68b3d9a7388edd to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e67fba14620f4143ad68b3d9a7388edd 2024-11-16T05:40:32,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/48cf9404a3c448f6b629aad6d8c2bacc to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/48cf9404a3c448f6b629aad6d8c2bacc 2024-11-16T05:40:32,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e71aea2cf98d4121b0286d25f28a5002 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e71aea2cf98d4121b0286d25f28a5002 2024-11-16T05:40:32,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/ed5e0fb9d8504466bdee5b38cabf0981 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/ed5e0fb9d8504466bdee5b38cabf0981 2024-11-16T05:40:32,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/1f3cc7208ace450a85c15cbd573499c4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/1f3cc7208ace450a85c15cbd573499c4 2024-11-16T05:40:32,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/cc768c28c7f841feb3fff3a815273291 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/cc768c28c7f841feb3fff3a815273291 2024-11-16T05:40:32,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/de7fd635a12149528b7c4c65d0a43ab8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/de7fd635a12149528b7c4c65d0a43ab8 2024-11-16T05:40:32,676 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/recovered.edits/425.seqid, newMaxSeqId=425, maxSeqId=4 2024-11-16T05:40:32,677 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. 2024-11-16T05:40:32,677 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 82b1d437a2b8fce4a136c21be6257221: Waiting for close lock at 1731735630023Running coprocessor pre-close hooks at 1731735630023Disabling compacts and flushes for region at 1731735630023Disabling writes for close at 1731735631329 (+1306 ms)Obtaining lock to block concurrent updates at 1731735631329Preparing flush snapshotting stores in 82b1d437a2b8fce4a136c21be6257221 at 1731735631329Finished memstore snapshotting TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731735631330 (+1 ms)Flushing stores of TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221. at 1731735631330Flushing 82b1d437a2b8fce4a136c21be6257221/A: creating writer at 1731735631330Flushing mob file 82b1d437a2b8fce4a136c21be6257221/A: appending metadata at 1731735631336 (+6 ms)Flushing mob file 82b1d437a2b8fce4a136c21be6257221/A: closing flushed file at 1731735631336Flushing 82b1d437a2b8fce4a136c21be6257221/A: appending metadata at 1731735631747 (+411 ms)Flushing 82b1d437a2b8fce4a136c21be6257221/A: closing flushed file at 1731735631747Flushing 82b1d437a2b8fce4a136c21be6257221/B: creating writer at 1731735631758 (+11 ms)Flushing 82b1d437a2b8fce4a136c21be6257221/B: appending metadata at 1731735631760 (+2 ms)Flushing 82b1d437a2b8fce4a136c21be6257221/B: closing flushed file at 1731735631760Flushing 82b1d437a2b8fce4a136c21be6257221/C: creating writer at 1731735632169 (+409 ms)Flushing 82b1d437a2b8fce4a136c21be6257221/C: appending metadata at 1731735632172 (+3 ms)Flushing 82b1d437a2b8fce4a136c21be6257221/C: closing flushed file at 1731735632172Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@663e41c4: reopening flushed file at 1731735632586 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@511faaae: reopening flushed file at 1731735632593 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a336a17: reopening flushed file at 1731735632599 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 82b1d437a2b8fce4a136c21be6257221 in 1275ms, sequenceid=422, compaction requested=true at 1731735632604 (+5 ms)Writing region close event to WAL at 1731735632673 (+69 ms)Running coprocessor post-close hooks at 1731735632676 (+3 ms)Closed at 1731735632677 (+1 ms) 2024-11-16T05:40:32,678 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:32,679 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=82b1d437a2b8fce4a136c21be6257221, regionState=CLOSED 2024-11-16T05:40:32,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:40:32,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-16T05:40:32,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 82b1d437a2b8fce4a136c21be6257221, server=3456ee6a3164,35555,1731735575219 in 2.8090 sec 2024-11-16T05:40:32,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-16T05:40:32,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b1d437a2b8fce4a136c21be6257221, UNASSIGN in 2.8140 sec 2024-11-16T05:40:32,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-16T05:40:32,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8170 sec 2024-11-16T05:40:32,685 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735632685"}]},"ts":"1731735632685"} 2024-11-16T05:40:32,687 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:40:32,687 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:40:32,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.8270 sec 2024-11-16T05:40:33,555 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:40:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:40:34,038 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:34,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:40:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:34,041 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T05:40:34,042 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:34,048 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,051 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/recovered.edits] 2024-11-16T05:40:34,054 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/1201fa175e04478d9302f1b3b5a46692 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/1201fa175e04478d9302f1b3b5a46692 2024-11-16T05:40:34,055 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3c9baaf74f4e476a857f33062fa01e86 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/3c9baaf74f4e476a857f33062fa01e86 2024-11-16T05:40:34,056 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/bee9efd1326f4b2bb2fdf37b9aa1dc77 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/bee9efd1326f4b2bb2fdf37b9aa1dc77 2024-11-16T05:40:34,058 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/d036b6d372a44f158c799c5343f267db to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/A/d036b6d372a44f158c799c5343f267db 2024-11-16T05:40:34,060 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/352d9b600d8d44bfa7805b3f814fa131 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/352d9b600d8d44bfa7805b3f814fa131 2024-11-16T05:40:34,061 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/5e994e86c1a8442a932b0ef84addb369 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/5e994e86c1a8442a932b0ef84addb369 2024-11-16T05:40:34,062 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/7dbad459bfa047978850434b09acac21 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/7dbad459bfa047978850434b09acac21 2024-11-16T05:40:34,063 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/95c7ecdb7dbb424d95d76215b1a870bc to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/B/95c7ecdb7dbb424d95d76215b1a870bc 2024-11-16T05:40:34,065 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/05e17ee5334c4714b064bb5c7146e729 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/05e17ee5334c4714b064bb5c7146e729 2024-11-16T05:40:34,066 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9a00464b814744849c119317df8f4ce7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/9a00464b814744849c119317df8f4ce7 2024-11-16T05:40:34,067 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/c461620d3bd44413a2e9778e338ab047 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/c461620d3bd44413a2e9778e338ab047 2024-11-16T05:40:34,068 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e3e33b5d306d4cfe8049d9177a821b58 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/C/e3e33b5d306d4cfe8049d9177a821b58 2024-11-16T05:40:34,071 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/recovered.edits/425.seqid to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221/recovered.edits/425.seqid 2024-11-16T05:40:34,072 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,072 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:40:34,072 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:40:34,073 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T05:40:34,078 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161003b4bf8125451d9fe8da3e431e53ca_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161003b4bf8125451d9fe8da3e431e53ca_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,080 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163200aad208c247a1b2731e691d2e4018_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163200aad208c247a1b2731e691d2e4018_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,081 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163bf7b4c0b96a4f869627bfddb339fcd5_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163bf7b4c0b96a4f869627bfddb339fcd5_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,083 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111655bd84258df849c595ba0f40bd98528d_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111655bd84258df849c595ba0f40bd98528d_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,084 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165db83a7873444c04a8de5f63dc2e2af5_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165db83a7873444c04a8de5f63dc2e2af5_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,085 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111663386aff40e34366811fd067b637be26_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111663386aff40e34366811fd067b637be26_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,087 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166efd92ad3e87400887bc9aa37fa10ae5_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166efd92ad3e87400887bc9aa37fa10ae5_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,088 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116729adfde791d453592da9d62d8bd8d45_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116729adfde791d453592da9d62d8bd8d45_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,089 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167a8baad1540d4f67b3ac69e175ba3c45_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167a8baad1540d4f67b3ac69e175ba3c45_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,091 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116880c63a76e1d444798802c9a42e160a6_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116880c63a76e1d444798802c9a42e160a6_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,092 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a5582e9ac2d6430aa11aca5efdd5dec4_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a5582e9ac2d6430aa11aca5efdd5dec4_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,093 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ad20812dd4664c7cb912d14839362abd_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ad20812dd4664c7cb912d14839362abd_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,095 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b1507727179b4e0eac6b73161e27fb49_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b1507727179b4e0eac6b73161e27fb49_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,096 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b3690f85270548fd8458e3dfe82f6068_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b3690f85270548fd8458e3dfe82f6068_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,097 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d70227feee774e3d828c237f672ee244_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d70227feee774e3d828c237f672ee244_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,099 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dafa67dd84ae4267811d380f65fedb5d_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dafa67dd84ae4267811d380f65fedb5d_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,100 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dc94bdb702824881a4c7f3d7b710cefb_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dc94bdb702824881a4c7f3d7b710cefb_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,101 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116df9b45808cdc48f48f1ff9d8b5fe45cb_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116df9b45808cdc48f48f1ff9d8b5fe45cb_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,102 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f00a9fdbecc64468a5131c5e535b5bf9_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f00a9fdbecc64468a5131c5e535b5bf9_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,104 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f50e59bf893d4353bf4d273a703568c5_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f50e59bf893d4353bf4d273a703568c5_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,105 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fd557a006ad84032b3a220038cc256a3_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fd557a006ad84032b3a220038cc256a3_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,106 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ff6aec366fde40ae9f2f5979e7f2c3fa_82b1d437a2b8fce4a136c21be6257221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ff6aec366fde40ae9f2f5979e7f2c3fa_82b1d437a2b8fce4a136c21be6257221 2024-11-16T05:40:34,107 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:40:34,109 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:34,112 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:40:34,114 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:40:34,115 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:34,115 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:40:34,115 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731735634115"}]},"ts":"9223372036854775807"} 2024-11-16T05:40:34,118 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:40:34,118 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 82b1d437a2b8fce4a136c21be6257221, NAME => 'TestAcidGuarantees,,1731735605401.82b1d437a2b8fce4a136c21be6257221.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:40:34,118 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:40:34,118 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731735634118"}]},"ts":"9223372036854775807"} 2024-11-16T05:40:34,120 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:40:34,121 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:34,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-11-16T05:40:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T05:40:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:40:34,148 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:34,161 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1185263931_22 at /127.0.0.1:35956 [Waiting for operation #279] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/cluster_d224ca95-f133-78db-847d-de6e87cd9c78/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1185263931_22 at /127.0.0.1:43986 [Waiting for operation #620] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1836729063_22 at /127.0.0.1:47358 [Waiting for operation #608] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/cluster_d224ca95-f133-78db-847d-de6e87cd9c78/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=578 (was 507) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3680 (was 3559) - AvailableMemoryMB LEAK? - 2024-11-16T05:40:34,170 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=578, ProcessCount=11, AvailableMemoryMB=3680 2024-11-16T05:40:34,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:40:34,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:40:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:34,174 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:40:34,174 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:34,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-16T05:40:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T05:40:34,175 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:40:34,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742080_1256 (size=959) 2024-11-16T05:40:34,182 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d 2024-11-16T05:40:34,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742081_1257 (size=53) 2024-11-16T05:40:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T05:40:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T05:40:34,592 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:40:34,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e8755229bb6a5d7b53307633970aa7aa, disabling compactions & flushes 2024-11-16T05:40:34,593 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:34,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:34,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. after waiting 0 ms 2024-11-16T05:40:34,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:34,593 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:34,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e8755229bb6a5d7b53307633970aa7aa: Waiting for close lock at 1731735634593Disabling compacts and flushes for region at 1731735634593Disabling writes for close at 1731735634593Writing region close event to WAL at 1731735634593Closed at 1731735634593 2024-11-16T05:40:34,596 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:40:34,597 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731735634596"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731735634596"}]},"ts":"1731735634596"} 2024-11-16T05:40:34,599 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:40:34,601 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:40:34,601 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735634601"}]},"ts":"1731735634601"} 2024-11-16T05:40:34,602 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:40:34,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8755229bb6a5d7b53307633970aa7aa, ASSIGN}] 2024-11-16T05:40:34,604 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8755229bb6a5d7b53307633970aa7aa, ASSIGN 2024-11-16T05:40:34,605 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8755229bb6a5d7b53307633970aa7aa, ASSIGN; state=OFFLINE, location=3456ee6a3164,35555,1731735575219; forceNewPlan=false, retain=false 2024-11-16T05:40:34,756 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e8755229bb6a5d7b53307633970aa7aa, regionState=OPENING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:34,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8755229bb6a5d7b53307633970aa7aa, ASSIGN because future has completed 2024-11-16T05:40:34,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:40:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T05:40:34,913 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:34,913 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:40:34,913 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,913 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:40:34,914 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,914 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,915 INFO [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,916 INFO [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8755229bb6a5d7b53307633970aa7aa columnFamilyName A 2024-11-16T05:40:34,916 DEBUG [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:34,917 INFO [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] regionserver.HStore(327): Store=e8755229bb6a5d7b53307633970aa7aa/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:40:34,917 INFO [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,918 INFO [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8755229bb6a5d7b53307633970aa7aa columnFamilyName B 2024-11-16T05:40:34,918 DEBUG [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:34,919 INFO [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] regionserver.HStore(327): Store=e8755229bb6a5d7b53307633970aa7aa/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:40:34,919 INFO [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,920 INFO [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8755229bb6a5d7b53307633970aa7aa columnFamilyName C 2024-11-16T05:40:34,920 DEBUG [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:40:34,921 INFO [StoreOpener-e8755229bb6a5d7b53307633970aa7aa-1 {}] regionserver.HStore(327): Store=e8755229bb6a5d7b53307633970aa7aa/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:40:34,921 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,922 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,922 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,923 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,923 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,924 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:40:34,925 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,928 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:40:34,929 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened e8755229bb6a5d7b53307633970aa7aa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73079281, jitterRate=0.08896614611148834}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:40:34,929 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:34,930 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for e8755229bb6a5d7b53307633970aa7aa: Running coprocessor pre-open hook at 1731735634914Writing region info on filesystem at 1731735634914Initializing all the Stores at 1731735634915 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735634915Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735634915Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735634915Cleaning up temporary data from old regions at 1731735634923 (+8 ms)Running coprocessor post-open hooks at 1731735634929 (+6 ms)Region opened successfully at 1731735634930 (+1 ms) 2024-11-16T05:40:34,931 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., pid=65, masterSystemTime=1731735634909 2024-11-16T05:40:34,932 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:34,932 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:34,933 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e8755229bb6a5d7b53307633970aa7aa, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:34,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:40:34,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-16T05:40:34,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 in 178 msec 2024-11-16T05:40:34,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-16T05:40:34,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8755229bb6a5d7b53307633970aa7aa, ASSIGN in 334 msec 2024-11-16T05:40:34,939 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:40:34,940 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735634940"}]},"ts":"1731735634940"} 2024-11-16T05:40:34,941 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:40:34,943 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:40:34,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 771 msec 2024-11-16T05:40:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T05:40:35,308 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:35,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,311 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@630ccb56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,313 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4233481c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,314 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,317 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:40:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,39409,1731735574536 2024-11-16T05:40:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4fc7e4db 2024-11-16T05:40:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:40:35,319 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:40:35,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,322 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@493b1746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,324 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3edd9ff0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,326 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b8a678, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,328 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db2b979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,330 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127eed2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,332 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705bc27b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,335 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@650440d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,337 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df0b32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,339 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,339 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee839b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,341 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a417210, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,344 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd44367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,345 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f4b05f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,348 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af122c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,350 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4599ee80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,351 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,352 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29fede41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,353 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a8d742, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,355 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ed4fa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,357 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f4ca0eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:40:35,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:40:35,358 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:40:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:40:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:40:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9da4d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:40:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:40:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:35,360 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:40:35,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de8d8d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:40:35,361 DEBUG [Thread-1191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,361 DEBUG [Thread-1187 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,361 DEBUG [Thread-1202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,361 DEBUG [Thread-1194 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:35,361 DEBUG [Thread-1198 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,361 DEBUG [Thread-1200 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,362 DEBUG [Thread-1196 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,362 DEBUG [Thread-1183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,363 DEBUG [Thread-1185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,363 DEBUG [Thread-1189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:40:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-16T05:40:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,365 DEBUG [Thread-1196 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:40:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:40:35,365 DEBUG [Thread-1196 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:40:35,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:40:35,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:40:35,366 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:40:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:40:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:40:35,367 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:35,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:40:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:40:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:40:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:40:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:40:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:35,375 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:35,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T05:40:35,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/d2cf692e3ebd45c083f538b339aa393a is 50, key is test_row_0/A:col10/1731735635370/Put/seqid=0 2024-11-16T05:40:35,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735695407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735695407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735695409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735695411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735695409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742082_1258 (size=12001) 2024-11-16T05:40:35,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/d2cf692e3ebd45c083f538b339aa393a 2024-11-16T05:40:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:40:35,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/e6818e904a7a4449ac825c72498ea43a is 50, key is test_row_0/B:col10/1731735635370/Put/seqid=0 2024-11-16T05:40:35,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742083_1259 (size=12001) 2024-11-16T05:40:35,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/e6818e904a7a4449ac825c72498ea43a 2024-11-16T05:40:35,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735695518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735695518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735695519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735695519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735695519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T05:40:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:35,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:35,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:35,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/5922202fe711438385f33df1b530a8a6 is 50, key is test_row_0/C:col10/1731735635370/Put/seqid=0 2024-11-16T05:40:35,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742084_1260 (size=12001) 2024-11-16T05:40:35,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/5922202fe711438385f33df1b530a8a6 2024-11-16T05:40:35,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/d2cf692e3ebd45c083f538b339aa393a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/d2cf692e3ebd45c083f538b339aa393a 2024-11-16T05:40:35,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/d2cf692e3ebd45c083f538b339aa393a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T05:40:35,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/e6818e904a7a4449ac825c72498ea43a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e6818e904a7a4449ac825c72498ea43a 2024-11-16T05:40:35,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e6818e904a7a4449ac825c72498ea43a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T05:40:35,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/5922202fe711438385f33df1b530a8a6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/5922202fe711438385f33df1b530a8a6 2024-11-16T05:40:35,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/5922202fe711438385f33df1b530a8a6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T05:40:35,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e8755229bb6a5d7b53307633970aa7aa in 232ms, sequenceid=13, compaction requested=false 2024-11-16T05:40:35,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:35,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T05:40:35,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:35,677 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T05:40:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:40:35,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/58f660c70834476daaf5a7949676ea16 is 50, key is test_row_0/A:col10/1731735635409/Put/seqid=0 2024-11-16T05:40:35,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742085_1261 (size=12001) 2024-11-16T05:40:35,688 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/58f660c70834476daaf5a7949676ea16 2024-11-16T05:40:35,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/ef541eb9c7844f5fa12029668741db3a is 50, key is test_row_0/B:col10/1731735635409/Put/seqid=0 2024-11-16T05:40:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:35,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:35,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742086_1262 (size=12001) 2024-11-16T05:40:35,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735695739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735695744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735695745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735695745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735695746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735695848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735695858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735695858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735695858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735695858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:40:36,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735696058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735696068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735696069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735696069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735696069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,140 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/ef541eb9c7844f5fa12029668741db3a 2024-11-16T05:40:36,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/d0d3f2df24cb44099b5ba8bf075c02b0 is 50, key is test_row_0/C:col10/1731735635409/Put/seqid=0 2024-11-16T05:40:36,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742087_1263 (size=12001) 2024-11-16T05:40:36,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735696368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735696378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735696379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735696379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735696379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:40:36,552 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/d0d3f2df24cb44099b5ba8bf075c02b0 2024-11-16T05:40:36,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/58f660c70834476daaf5a7949676ea16 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/58f660c70834476daaf5a7949676ea16 2024-11-16T05:40:36,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/58f660c70834476daaf5a7949676ea16, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T05:40:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/ef541eb9c7844f5fa12029668741db3a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ef541eb9c7844f5fa12029668741db3a 2024-11-16T05:40:36,596 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ef541eb9c7844f5fa12029668741db3a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T05:40:36,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/d0d3f2df24cb44099b5ba8bf075c02b0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d0d3f2df24cb44099b5ba8bf075c02b0 2024-11-16T05:40:36,609 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d0d3f2df24cb44099b5ba8bf075c02b0, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T05:40:36,611 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for e8755229bb6a5d7b53307633970aa7aa in 934ms, sequenceid=37, compaction requested=false 2024-11-16T05:40:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-16T05:40:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-16T05:40:36,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-16T05:40:36,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2460 sec 2024-11-16T05:40:36,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.2570 sec 2024-11-16T05:40:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:36,881 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:36,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T05:40:36,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/322b9ee3a9fe4fae9bf79fd9d298532a is 50, key is test_row_0/A:col10/1731735635744/Put/seqid=0 2024-11-16T05:40:36,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742088_1264 (size=12001) 2024-11-16T05:40:36,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735696901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735696903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735696904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735696906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735696907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735697018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735697018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735697018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735697018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735697018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735697228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735697228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735697228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735697228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735697229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/322b9ee3a9fe4fae9bf79fd9d298532a 2024-11-16T05:40:37,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/56afb1e949d44d6887e2d60616da8848 is 50, key is test_row_0/B:col10/1731735635744/Put/seqid=0 2024-11-16T05:40:37,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742089_1265 (size=12001) 2024-11-16T05:40:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:40:37,508 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:37,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-16T05:40:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:40:37,510 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:37,511 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:37,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735697538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735697538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735697538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735697539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735697539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:40:37,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:40:37,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:37,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:37,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:37,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:37,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:37,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/56afb1e949d44d6887e2d60616da8848 2024-11-16T05:40:37,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/b9044d18ce414fa89ef2b96babc57898 is 50, key is test_row_0/C:col10/1731735635744/Put/seqid=0 2024-11-16T05:40:37,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742090_1266 (size=12001) 2024-11-16T05:40:37,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/b9044d18ce414fa89ef2b96babc57898 2024-11-16T05:40:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/322b9ee3a9fe4fae9bf79fd9d298532a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/322b9ee3a9fe4fae9bf79fd9d298532a 2024-11-16T05:40:37,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/322b9ee3a9fe4fae9bf79fd9d298532a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T05:40:37,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/56afb1e949d44d6887e2d60616da8848 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/56afb1e949d44d6887e2d60616da8848 2024-11-16T05:40:37,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/56afb1e949d44d6887e2d60616da8848, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T05:40:37,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/b9044d18ce414fa89ef2b96babc57898 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b9044d18ce414fa89ef2b96babc57898 2024-11-16T05:40:37,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b9044d18ce414fa89ef2b96babc57898, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T05:40:37,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e8755229bb6a5d7b53307633970aa7aa in 898ms, sequenceid=51, compaction requested=true 2024-11-16T05:40:37,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:37,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:37,779 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:37,780 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:37,781 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:37,781 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:37,781 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/A is initiating minor compaction (all files) 2024-11-16T05:40:37,781 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/B is initiating minor compaction (all files) 2024-11-16T05:40:37,781 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/A in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:37,781 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/B in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:37,781 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e6818e904a7a4449ac825c72498ea43a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ef541eb9c7844f5fa12029668741db3a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/56afb1e949d44d6887e2d60616da8848] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=35.2 K 2024-11-16T05:40:37,781 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/d2cf692e3ebd45c083f538b339aa393a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/58f660c70834476daaf5a7949676ea16, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/322b9ee3a9fe4fae9bf79fd9d298532a] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=35.2 K 2024-11-16T05:40:37,781 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e6818e904a7a4449ac825c72498ea43a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731735635370 2024-11-16T05:40:37,781 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2cf692e3ebd45c083f538b339aa393a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731735635370 2024-11-16T05:40:37,782 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58f660c70834476daaf5a7949676ea16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731735635405 2024-11-16T05:40:37,782 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ef541eb9c7844f5fa12029668741db3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731735635405 2024-11-16T05:40:37,782 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 322b9ee3a9fe4fae9bf79fd9d298532a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731735635744 2024-11-16T05:40:37,782 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 56afb1e949d44d6887e2d60616da8848, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731735635744 2024-11-16T05:40:37,800 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#A#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:37,800 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:37,801 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/e4dd9242a10344da84144c1b87a81936 is 50, key is test_row_0/B:col10/1731735635744/Put/seqid=0 2024-11-16T05:40:37,801 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/cb232d62af6a466ca7ef14974e8b2774 is 50, key is test_row_0/A:col10/1731735635744/Put/seqid=0 2024-11-16T05:40:37,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:40:37,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:37,816 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T05:40:37,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742091_1267 (size=12104) 2024-11-16T05:40:37,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/cc1f8be1997b49248b454e3ff4c6c420 is 50, key is test_row_0/A:col10/1731735636906/Put/seqid=0 2024-11-16T05:40:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:40:37,829 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/cb232d62af6a466ca7ef14974e8b2774 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cb232d62af6a466ca7ef14974e8b2774 2024-11-16T05:40:37,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742092_1268 (size=12104) 2024-11-16T05:40:37,850 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/e4dd9242a10344da84144c1b87a81936 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e4dd9242a10344da84144c1b87a81936 2024-11-16T05:40:37,863 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/B of e8755229bb6a5d7b53307633970aa7aa into e4dd9242a10344da84144c1b87a81936(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:37,863 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:37,863 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/B, priority=13, startTime=1731735637780; duration=0sec 2024-11-16T05:40:37,863 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:37,863 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:B 2024-11-16T05:40:37,863 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:37,864 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:37,864 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/C is initiating minor compaction (all files) 2024-11-16T05:40:37,864 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/C in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:37,864 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/5922202fe711438385f33df1b530a8a6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d0d3f2df24cb44099b5ba8bf075c02b0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b9044d18ce414fa89ef2b96babc57898] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=35.2 K 2024-11-16T05:40:37,865 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5922202fe711438385f33df1b530a8a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731735635370 2024-11-16T05:40:37,865 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting d0d3f2df24cb44099b5ba8bf075c02b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731735635405 2024-11-16T05:40:37,866 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting b9044d18ce414fa89ef2b96babc57898, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731735635744 2024-11-16T05:40:37,880 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/A of e8755229bb6a5d7b53307633970aa7aa into cb232d62af6a466ca7ef14974e8b2774(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:37,880 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:37,881 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/A, priority=13, startTime=1731735637779; duration=0sec 2024-11-16T05:40:37,881 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:37,881 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:A 2024-11-16T05:40:37,882 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#C#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:37,883 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/1b3bc292891b40b4b8ed4a8d22495ff0 is 50, key is test_row_0/C:col10/1731735635744/Put/seqid=0 2024-11-16T05:40:37,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742093_1269 (size=12001) 2024-11-16T05:40:37,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742094_1270 (size=12104) 2024-11-16T05:40:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:38,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:38,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735698059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735698060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735698061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735698062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735698066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:40:38,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735698168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735698168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735698178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735698178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735698178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,200 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:40:38,293 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/cc1f8be1997b49248b454e3ff4c6c420 2024-11-16T05:40:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/60b8167bd380489eaf0a9a09e0ec89a3 is 50, key is test_row_0/B:col10/1731735636906/Put/seqid=0 2024-11-16T05:40:38,327 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/1b3bc292891b40b4b8ed4a8d22495ff0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1b3bc292891b40b4b8ed4a8d22495ff0 2024-11-16T05:40:38,334 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/C of e8755229bb6a5d7b53307633970aa7aa into 1b3bc292891b40b4b8ed4a8d22495ff0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:38,334 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:38,334 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/C, priority=13, startTime=1731735637780; duration=0sec 2024-11-16T05:40:38,334 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:38,334 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:C 2024-11-16T05:40:38,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742095_1271 (size=12001) 2024-11-16T05:40:38,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735698379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735698379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735698388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735698388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735698388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:40:38,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735698688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735698688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735698698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735698698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735698698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:38,737 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/60b8167bd380489eaf0a9a09e0ec89a3 2024-11-16T05:40:38,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/edcf44b1a19c4cd088daa38b5f1db23f is 50, key is test_row_0/C:col10/1731735636906/Put/seqid=0 2024-11-16T05:40:38,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742096_1272 (size=12001) 2024-11-16T05:40:38,749 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/edcf44b1a19c4cd088daa38b5f1db23f 2024-11-16T05:40:38,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/cc1f8be1997b49248b454e3ff4c6c420 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cc1f8be1997b49248b454e3ff4c6c420 2024-11-16T05:40:38,757 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cc1f8be1997b49248b454e3ff4c6c420, entries=150, sequenceid=73, filesize=11.7 K 2024-11-16T05:40:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/60b8167bd380489eaf0a9a09e0ec89a3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/60b8167bd380489eaf0a9a09e0ec89a3 2024-11-16T05:40:38,761 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/60b8167bd380489eaf0a9a09e0ec89a3, entries=150, sequenceid=73, filesize=11.7 K 2024-11-16T05:40:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/edcf44b1a19c4cd088daa38b5f1db23f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/edcf44b1a19c4cd088daa38b5f1db23f 2024-11-16T05:40:38,766 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/edcf44b1a19c4cd088daa38b5f1db23f, entries=150, sequenceid=73, filesize=11.7 K 2024-11-16T05:40:38,767 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e8755229bb6a5d7b53307633970aa7aa in 951ms, sequenceid=73, compaction requested=false 2024-11-16T05:40:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-16T05:40:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-16T05:40:38,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-16T05:40:38,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2570 sec 2024-11-16T05:40:38,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2620 sec 2024-11-16T05:40:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:39,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:39,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T05:40:39,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/ac380ccee4ce433c8cd436c1c3d20330 is 50, key is test_row_0/A:col10/1731735638064/Put/seqid=0 2024-11-16T05:40:39,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742097_1273 (size=14341) 2024-11-16T05:40:39,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735699221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735699226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735699225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735699227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735699228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735699338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735699339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735699339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735699339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735699339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735699548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735699548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735699549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735699549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735699549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/ac380ccee4ce433c8cd436c1c3d20330 2024-11-16T05:40:39,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/3c6026bf7b0046b998d6c980f3a7c5c3 is 50, key is test_row_0/B:col10/1731735638064/Put/seqid=0 2024-11-16T05:40:39,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742098_1274 (size=12001) 2024-11-16T05:40:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:40:39,658 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:39,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-16T05:40:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:40:39,660 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:39,661 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:39,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:40:39,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T05:40:39,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:39,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:39,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:39,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:39,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:39,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735699859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735699859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735699859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735699859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735699859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:39,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T05:40:39,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:39,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:39,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:39,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:39,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:40:40,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/3c6026bf7b0046b998d6c980f3a7c5c3 2024-11-16T05:40:40,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/ead3b1833daf4d10810a7b533c828000 is 50, key is test_row_0/C:col10/1731735638064/Put/seqid=0 2024-11-16T05:40:40,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742099_1275 (size=12001) 2024-11-16T05:40:40,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/ead3b1833daf4d10810a7b533c828000 2024-11-16T05:40:40,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/ac380ccee4ce433c8cd436c1c3d20330 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/ac380ccee4ce433c8cd436c1c3d20330 2024-11-16T05:40:40,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/ac380ccee4ce433c8cd436c1c3d20330, entries=200, sequenceid=92, filesize=14.0 K 2024-11-16T05:40:40,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/3c6026bf7b0046b998d6c980f3a7c5c3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3c6026bf7b0046b998d6c980f3a7c5c3 2024-11-16T05:40:40,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3c6026bf7b0046b998d6c980f3a7c5c3, entries=150, sequenceid=92, filesize=11.7 K 2024-11-16T05:40:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/ead3b1833daf4d10810a7b533c828000 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ead3b1833daf4d10810a7b533c828000 2024-11-16T05:40:40,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ead3b1833daf4d10810a7b533c828000, entries=150, sequenceid=92, filesize=11.7 K 2024-11-16T05:40:40,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for e8755229bb6a5d7b53307633970aa7aa in 862ms, sequenceid=92, compaction requested=true 2024-11-16T05:40:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:40,062 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:40,062 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:40,063 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:40,063 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/B is initiating minor compaction (all files) 2024-11-16T05:40:40,063 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/B in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:40,063 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e4dd9242a10344da84144c1b87a81936, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/60b8167bd380489eaf0a9a09e0ec89a3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3c6026bf7b0046b998d6c980f3a7c5c3] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=35.3 K 2024-11-16T05:40:40,064 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:40,064 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/A is initiating minor compaction (all files) 2024-11-16T05:40:40,064 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/A in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:40,064 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cb232d62af6a466ca7ef14974e8b2774, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cc1f8be1997b49248b454e3ff4c6c420, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/ac380ccee4ce433c8cd436c1c3d20330] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=37.5 K 2024-11-16T05:40:40,064 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e4dd9242a10344da84144c1b87a81936, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731735635744 2024-11-16T05:40:40,064 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb232d62af6a466ca7ef14974e8b2774, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731735635744 2024-11-16T05:40:40,065 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc1f8be1997b49248b454e3ff4c6c420, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731735636896 2024-11-16T05:40:40,065 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b8167bd380489eaf0a9a09e0ec89a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731735636896 2024-11-16T05:40:40,065 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac380ccee4ce433c8cd436c1c3d20330, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731735638051 2024-11-16T05:40:40,065 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c6026bf7b0046b998d6c980f3a7c5c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731735638064 2024-11-16T05:40:40,079 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:40,080 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/eeaf90d258404e879854818d0c7d4f00 is 50, key is test_row_0/A:col10/1731735638064/Put/seqid=0 2024-11-16T05:40:40,088 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:40,088 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/d8e282fc06344990b0444d10892b189b is 50, key is test_row_0/B:col10/1731735638064/Put/seqid=0 2024-11-16T05:40:40,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742100_1276 (size=12207) 2024-11-16T05:40:40,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742101_1277 (size=12207) 2024-11-16T05:40:40,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T05:40:40,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:40,123 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T05:40:40,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/9656687f86094f9e9f84558a6e76ba02 is 50, key is test_row_0/A:col10/1731735639220/Put/seqid=0 2024-11-16T05:40:40,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742102_1278 (size=12001) 2024-11-16T05:40:40,160 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/9656687f86094f9e9f84558a6e76ba02 2024-11-16T05:40:40,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/17c7aa3ba99546bd9bd9d73e5a710bd2 is 50, key is test_row_0/B:col10/1731735639220/Put/seqid=0 2024-11-16T05:40:40,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742103_1279 (size=12001) 2024-11-16T05:40:40,197 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/17c7aa3ba99546bd9bd9d73e5a710bd2 2024-11-16T05:40:40,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/ff56357e0a24426c85bc6d659519abfd is 50, key is test_row_0/C:col10/1731735639220/Put/seqid=0 2024-11-16T05:40:40,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742104_1280 (size=12001) 2024-11-16T05:40:40,241 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/ff56357e0a24426c85bc6d659519abfd 2024-11-16T05:40:40,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/9656687f86094f9e9f84558a6e76ba02 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/9656687f86094f9e9f84558a6e76ba02 2024-11-16T05:40:40,256 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/9656687f86094f9e9f84558a6e76ba02, entries=150, sequenceid=113, filesize=11.7 K 2024-11-16T05:40:40,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/17c7aa3ba99546bd9bd9d73e5a710bd2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/17c7aa3ba99546bd9bd9d73e5a710bd2 2024-11-16T05:40:40,264 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/17c7aa3ba99546bd9bd9d73e5a710bd2, entries=150, sequenceid=113, filesize=11.7 K 2024-11-16T05:40:40,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/ff56357e0a24426c85bc6d659519abfd as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ff56357e0a24426c85bc6d659519abfd 2024-11-16T05:40:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:40:40,296 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ff56357e0a24426c85bc6d659519abfd, entries=150, sequenceid=113, filesize=11.7 K 2024-11-16T05:40:40,297 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for e8755229bb6a5d7b53307633970aa7aa in 174ms, sequenceid=113, compaction requested=true 2024-11-16T05:40:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-16T05:40:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-16T05:40:40,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-16T05:40:40,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 638 msec 2024-11-16T05:40:40,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 644 msec 2024-11-16T05:40:40,378 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:40,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:40:40,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/b8b3f517dd534cb99b24ea896bf2f20e is 50, key is test_row_0/A:col10/1731735640376/Put/seqid=0 2024-11-16T05:40:40,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742105_1281 (size=14341) 2024-11-16T05:40:40,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735700407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735700407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735700408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735700409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735700412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,517 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/eeaf90d258404e879854818d0c7d4f00 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/eeaf90d258404e879854818d0c7d4f00 2024-11-16T05:40:40,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735700518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735700518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735700519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735700519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735700519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,525 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/d8e282fc06344990b0444d10892b189b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/d8e282fc06344990b0444d10892b189b 2024-11-16T05:40:40,526 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/A of e8755229bb6a5d7b53307633970aa7aa into eeaf90d258404e879854818d0c7d4f00(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:40,526 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:40,526 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/A, priority=13, startTime=1731735640062; duration=0sec 2024-11-16T05:40:40,526 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:40,526 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:A 2024-11-16T05:40:40,526 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:40,528 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:40,528 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/C is initiating minor compaction (all files) 2024-11-16T05:40:40,528 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/C in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:40,528 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1b3bc292891b40b4b8ed4a8d22495ff0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/edcf44b1a19c4cd088daa38b5f1db23f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ead3b1833daf4d10810a7b533c828000, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ff56357e0a24426c85bc6d659519abfd] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=47.0 K 2024-11-16T05:40:40,529 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b3bc292891b40b4b8ed4a8d22495ff0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731735635744 2024-11-16T05:40:40,530 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting edcf44b1a19c4cd088daa38b5f1db23f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731735636896 2024-11-16T05:40:40,530 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ead3b1833daf4d10810a7b533c828000, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731735638064 2024-11-16T05:40:40,530 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff56357e0a24426c85bc6d659519abfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731735639220 2024-11-16T05:40:40,531 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/B of e8755229bb6a5d7b53307633970aa7aa into d8e282fc06344990b0444d10892b189b(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:40,531 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:40,531 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/B, priority=13, startTime=1731735640062; duration=0sec 2024-11-16T05:40:40,532 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:40,532 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:B 2024-11-16T05:40:40,551 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:40,552 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/a6295ec500194dd3b31101c0abbf0bb6 is 50, key is test_row_0/C:col10/1731735639220/Put/seqid=0 2024-11-16T05:40:40,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742106_1282 (size=12241) 2024-11-16T05:40:40,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735700728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735700728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735700728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735700729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735700729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:40:40,798 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:40,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-16T05:40:40,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/b8b3f517dd534cb99b24ea896bf2f20e 2024-11-16T05:40:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T05:40:40,803 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:40,804 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:40,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:40,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/1985b67e2a744596aa652c50dda68203 is 50, key is test_row_0/B:col10/1731735640376/Put/seqid=0 2024-11-16T05:40:40,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742107_1283 (size=12001) 2024-11-16T05:40:40,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/1985b67e2a744596aa652c50dda68203 2024-11-16T05:40:40,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/2c234a1d3d584fd19bf0f2aca998f24d is 50, key is test_row_0/C:col10/1731735640376/Put/seqid=0 2024-11-16T05:40:40,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742108_1284 (size=12001) 2024-11-16T05:40:40,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/2c234a1d3d584fd19bf0f2aca998f24d 2024-11-16T05:40:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T05:40:40,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/b8b3f517dd534cb99b24ea896bf2f20e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b8b3f517dd534cb99b24ea896bf2f20e 2024-11-16T05:40:40,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b8b3f517dd534cb99b24ea896bf2f20e, entries=200, sequenceid=125, filesize=14.0 K 2024-11-16T05:40:40,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/1985b67e2a744596aa652c50dda68203 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1985b67e2a744596aa652c50dda68203 2024-11-16T05:40:40,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1985b67e2a744596aa652c50dda68203, entries=150, sequenceid=125, filesize=11.7 K 2024-11-16T05:40:40,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/2c234a1d3d584fd19bf0f2aca998f24d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2c234a1d3d584fd19bf0f2aca998f24d 2024-11-16T05:40:40,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2c234a1d3d584fd19bf0f2aca998f24d, entries=150, sequenceid=125, filesize=11.7 K 2024-11-16T05:40:40,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e8755229bb6a5d7b53307633970aa7aa in 563ms, sequenceid=125, compaction requested=true 2024-11-16T05:40:40,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:40,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:A, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:40,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:40,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:B, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:40,942 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:40,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:40,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:40,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:40:40,943 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:40,943 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/A is initiating minor compaction (all files) 2024-11-16T05:40:40,943 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/A in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:40,943 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/eeaf90d258404e879854818d0c7d4f00, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/9656687f86094f9e9f84558a6e76ba02, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b8b3f517dd534cb99b24ea896bf2f20e] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=37.6 K 2024-11-16T05:40:40,944 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting eeaf90d258404e879854818d0c7d4f00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731735638064 2024-11-16T05:40:40,944 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9656687f86094f9e9f84558a6e76ba02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731735639220 2024-11-16T05:40:40,945 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting b8b3f517dd534cb99b24ea896bf2f20e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731735640372 2024-11-16T05:40:40,958 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:40,959 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/067889a4678d4d9888dcfd44c1558248 is 50, key is test_row_0/A:col10/1731735640376/Put/seqid=0 2024-11-16T05:40:40,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T05:40:40,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:40,960 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T05:40:40,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/318aac580f8342f19d9bca74c5c28283 is 50, key is test_row_0/A:col10/1731735640400/Put/seqid=0 2024-11-16T05:40:40,995 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/a6295ec500194dd3b31101c0abbf0bb6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a6295ec500194dd3b31101c0abbf0bb6 2024-11-16T05:40:41,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742109_1285 (size=12309) 2024-11-16T05:40:41,003 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/C of e8755229bb6a5d7b53307633970aa7aa into a6295ec500194dd3b31101c0abbf0bb6(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:41,005 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:41,005 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/C, priority=12, startTime=1731735640062; duration=0sec 2024-11-16T05:40:41,005 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:40:41,005 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:C 2024-11-16T05:40:41,005 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:C 2024-11-16T05:40:41,005 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:41,006 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:41,007 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/B is initiating minor compaction (all files) 2024-11-16T05:40:41,007 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/B in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:41,007 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/d8e282fc06344990b0444d10892b189b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/17c7aa3ba99546bd9bd9d73e5a710bd2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1985b67e2a744596aa652c50dda68203] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=35.4 K 2024-11-16T05:40:41,007 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e282fc06344990b0444d10892b189b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731735638064 2024-11-16T05:40:41,008 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17c7aa3ba99546bd9bd9d73e5a710bd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731735639220 2024-11-16T05:40:41,008 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1985b67e2a744596aa652c50dda68203, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731735640376 2024-11-16T05:40:41,019 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/067889a4678d4d9888dcfd44c1558248 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/067889a4678d4d9888dcfd44c1558248 2024-11-16T05:40:41,028 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/A of e8755229bb6a5d7b53307633970aa7aa into 067889a4678d4d9888dcfd44c1558248(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:41,029 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:41,029 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/A, priority=13, startTime=1731735640942; duration=0sec 2024-11-16T05:40:41,029 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:41,029 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:A 2024-11-16T05:40:41,029 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T05:40:41,030 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T05:40:41,030 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T05:40:41,030 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. because compaction request was cancelled 2024-11-16T05:40:41,030 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:C 2024-11-16T05:40:41,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742110_1286 (size=12151) 2024-11-16T05:40:41,040 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/318aac580f8342f19d9bca74c5c28283 2024-11-16T05:40:41,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:41,043 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#B#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:41,043 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/131f7f88ef524e5c8c42a17d32a8bec6 is 50, key is test_row_0/B:col10/1731735640376/Put/seqid=0 2024-11-16T05:40:41,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735701053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735701053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735701054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735701055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735701055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/989dd8da1fd04291b56c1c4db134e83b is 50, key is test_row_0/B:col10/1731735640400/Put/seqid=0 2024-11-16T05:40:41,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742111_1287 (size=12309) 2024-11-16T05:40:41,087 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/131f7f88ef524e5c8c42a17d32a8bec6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/131f7f88ef524e5c8c42a17d32a8bec6 2024-11-16T05:40:41,095 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/B of e8755229bb6a5d7b53307633970aa7aa into 131f7f88ef524e5c8c42a17d32a8bec6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:41,095 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:41,095 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/B, priority=13, startTime=1731735640942; duration=0sec 2024-11-16T05:40:41,095 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:41,095 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:B 2024-11-16T05:40:41,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742112_1288 (size=12151) 2024-11-16T05:40:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T05:40:41,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735701168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735701169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735701169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735701169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735701169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735701378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735701378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735701378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735701378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735701379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T05:40:41,513 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/989dd8da1fd04291b56c1c4db134e83b 2024-11-16T05:40:41,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/a45b6a3777e54e0c8d1023f89f1467ea is 50, key is test_row_0/C:col10/1731735640400/Put/seqid=0 2024-11-16T05:40:41,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742113_1289 (size=12151) 2024-11-16T05:40:41,572 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/a45b6a3777e54e0c8d1023f89f1467ea 2024-11-16T05:40:41,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/318aac580f8342f19d9bca74c5c28283 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/318aac580f8342f19d9bca74c5c28283 2024-11-16T05:40:41,583 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/318aac580f8342f19d9bca74c5c28283, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T05:40:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/989dd8da1fd04291b56c1c4db134e83b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/989dd8da1fd04291b56c1c4db134e83b 2024-11-16T05:40:41,591 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/989dd8da1fd04291b56c1c4db134e83b, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T05:40:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/a45b6a3777e54e0c8d1023f89f1467ea as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a45b6a3777e54e0c8d1023f89f1467ea 2024-11-16T05:40:41,599 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a45b6a3777e54e0c8d1023f89f1467ea, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T05:40:41,601 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for e8755229bb6a5d7b53307633970aa7aa in 641ms, sequenceid=151, compaction requested=true 2024-11-16T05:40:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-16T05:40:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-16T05:40:41,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-16T05:40:41,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 799 msec 2024-11-16T05:40:41,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 809 msec 2024-11-16T05:40:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:41,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T05:40:41,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/0af5c97aae354cfda7d09a123cc58586 is 50, key is test_row_0/A:col10/1731735641054/Put/seqid=0 2024-11-16T05:40:41,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735701740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735701749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735701750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742114_1290 (size=12151) 2024-11-16T05:40:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735701752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/0af5c97aae354cfda7d09a123cc58586 2024-11-16T05:40:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735701750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/2b0b3fdafb7648a9b8150b29e2d45f70 is 50, key is test_row_0/B:col10/1731735641054/Put/seqid=0 2024-11-16T05:40:41,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742115_1291 (size=12151) 2024-11-16T05:40:41,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735701858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735701869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735701869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735701869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735701878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T05:40:41,938 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:41,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-16T05:40:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:40:41,942 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:41,942 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:41,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:40:42,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735702069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735702078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735702078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735702088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735702088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:40:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:40:42,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:42,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/2b0b3fdafb7648a9b8150b29e2d45f70 2024-11-16T05:40:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:40:42,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/3b80a4425793443e94111f28c92a9403 is 50, key is test_row_0/C:col10/1731735641054/Put/seqid=0 2024-11-16T05:40:42,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742116_1292 (size=12151) 2024-11-16T05:40:42,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/3b80a4425793443e94111f28c92a9403 2024-11-16T05:40:42,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/0af5c97aae354cfda7d09a123cc58586 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/0af5c97aae354cfda7d09a123cc58586 2024-11-16T05:40:42,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/0af5c97aae354cfda7d09a123cc58586, entries=150, sequenceid=169, filesize=11.9 K 2024-11-16T05:40:42,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/2b0b3fdafb7648a9b8150b29e2d45f70 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/2b0b3fdafb7648a9b8150b29e2d45f70 2024-11-16T05:40:42,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/2b0b3fdafb7648a9b8150b29e2d45f70, entries=150, sequenceid=169, filesize=11.9 K 2024-11-16T05:40:42,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/3b80a4425793443e94111f28c92a9403 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3b80a4425793443e94111f28c92a9403 2024-11-16T05:40:42,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3b80a4425793443e94111f28c92a9403, entries=150, sequenceid=169, filesize=11.9 K 2024-11-16T05:40:42,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e8755229bb6a5d7b53307633970aa7aa in 627ms, sequenceid=169, compaction requested=true 2024-11-16T05:40:42,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:42,319 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:42,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:42,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:42,319 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:42,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:42,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:42,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:42,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:42,320 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:42,320 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/A is initiating minor compaction (all files) 2024-11-16T05:40:42,320 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/A in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,320 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:42,320 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/067889a4678d4d9888dcfd44c1558248, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/318aac580f8342f19d9bca74c5c28283, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/0af5c97aae354cfda7d09a123cc58586] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=35.8 K 2024-11-16T05:40:42,320 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/B is initiating minor compaction (all files) 2024-11-16T05:40:42,320 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/B in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,320 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/131f7f88ef524e5c8c42a17d32a8bec6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/989dd8da1fd04291b56c1c4db134e83b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/2b0b3fdafb7648a9b8150b29e2d45f70] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=35.8 K 2024-11-16T05:40:42,320 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 131f7f88ef524e5c8c42a17d32a8bec6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731735640376 2024-11-16T05:40:42,321 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 067889a4678d4d9888dcfd44c1558248, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731735640376 2024-11-16T05:40:42,321 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 989dd8da1fd04291b56c1c4db134e83b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731735640400 2024-11-16T05:40:42,321 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b0b3fdafb7648a9b8150b29e2d45f70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731735641052 2024-11-16T05:40:42,321 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 318aac580f8342f19d9bca74c5c28283, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731735640400 2024-11-16T05:40:42,322 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af5c97aae354cfda7d09a123cc58586, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731735641052 2024-11-16T05:40:42,330 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#A#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:42,331 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/5c9804378e4c4980a1322223e88299b6 is 50, key is test_row_0/A:col10/1731735641054/Put/seqid=0 2024-11-16T05:40:42,331 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:42,332 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/3e7a91a270f34fb6b29e5a9cfd7e0ed1 is 50, key is test_row_0/B:col10/1731735641054/Put/seqid=0 2024-11-16T05:40:42,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742118_1294 (size=12561) 2024-11-16T05:40:42,357 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/5c9804378e4c4980a1322223e88299b6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/5c9804378e4c4980a1322223e88299b6 2024-11-16T05:40:42,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742117_1293 (size=12561) 2024-11-16T05:40:42,362 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/A of e8755229bb6a5d7b53307633970aa7aa into 5c9804378e4c4980a1322223e88299b6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:42,362 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:42,362 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/A, priority=13, startTime=1731735642318; duration=0sec 2024-11-16T05:40:42,362 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:42,362 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:A 2024-11-16T05:40:42,363 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:42,365 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:42,365 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/C is initiating minor compaction (all files) 2024-11-16T05:40:42,365 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/C in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,365 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a6295ec500194dd3b31101c0abbf0bb6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2c234a1d3d584fd19bf0f2aca998f24d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a45b6a3777e54e0c8d1023f89f1467ea, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3b80a4425793443e94111f28c92a9403] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=47.4 K 2024-11-16T05:40:42,366 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6295ec500194dd3b31101c0abbf0bb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731735639220 2024-11-16T05:40:42,367 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c234a1d3d584fd19bf0f2aca998f24d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731735640376 2024-11-16T05:40:42,369 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/3e7a91a270f34fb6b29e5a9cfd7e0ed1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3e7a91a270f34fb6b29e5a9cfd7e0ed1 2024-11-16T05:40:42,370 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a45b6a3777e54e0c8d1023f89f1467ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731735640400 2024-11-16T05:40:42,371 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b80a4425793443e94111f28c92a9403, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731735641052 2024-11-16T05:40:42,376 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/B of e8755229bb6a5d7b53307633970aa7aa into 3e7a91a270f34fb6b29e5a9cfd7e0ed1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:42,376 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:42,376 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/B, priority=13, startTime=1731735642319; duration=0sec 2024-11-16T05:40:42,377 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:42,377 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:B 2024-11-16T05:40:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:42,383 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:42,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T05:40:42,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/b45b2f04728a41de86cb37b71cfe4efd is 50, key is test_row_0/A:col10/1731735641740/Put/seqid=0 2024-11-16T05:40:42,392 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:42,393 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/3ade528461374f70b024cad06ef4d2da is 50, key is test_row_0/C:col10/1731735641054/Put/seqid=0 2024-11-16T05:40:42,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742119_1295 (size=14541) 2024-11-16T05:40:42,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742120_1296 (size=12527) 2024-11-16T05:40:42,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:40:42,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,404 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/3ade528461374f70b024cad06ef4d2da as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3ade528461374f70b024cad06ef4d2da 2024-11-16T05:40:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735702401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735702404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735702406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,413 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/C of e8755229bb6a5d7b53307633970aa7aa into 3ade528461374f70b024cad06ef4d2da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:42,413 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:42,413 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/C, priority=12, startTime=1731735642319; duration=0sec 2024-11-16T05:40:42,413 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:42,413 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:C 2024-11-16T05:40:42,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735702408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735702410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735702518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735702518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735702518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735702518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735702518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:40:42,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:42,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:40:42,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:40:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735702728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735702728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735702728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735702728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735702729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:42,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/b45b2f04728a41de86cb37b71cfe4efd 2024-11-16T05:40:42,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/002ee3f8edf44be99438a604476b9bf9 is 50, key is test_row_0/B:col10/1731735641740/Put/seqid=0 2024-11-16T05:40:42,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742121_1297 (size=12151) 2024-11-16T05:40:42,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/002ee3f8edf44be99438a604476b9bf9 2024-11-16T05:40:42,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/cfb9ecc59cf84718a97058e9c541b7a8 is 50, key is test_row_0/C:col10/1731735641740/Put/seqid=0 2024-11-16T05:40:42,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742122_1298 (size=12151) 2024-11-16T05:40:42,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/cfb9ecc59cf84718a97058e9c541b7a8 2024-11-16T05:40:42,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/b45b2f04728a41de86cb37b71cfe4efd as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b45b2f04728a41de86cb37b71cfe4efd 2024-11-16T05:40:42,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b45b2f04728a41de86cb37b71cfe4efd, entries=200, sequenceid=193, filesize=14.2 K 2024-11-16T05:40:42,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/002ee3f8edf44be99438a604476b9bf9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/002ee3f8edf44be99438a604476b9bf9 2024-11-16T05:40:42,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/002ee3f8edf44be99438a604476b9bf9, entries=150, sequenceid=193, filesize=11.9 K 2024-11-16T05:40:42,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:40:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:42,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:42,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/cfb9ecc59cf84718a97058e9c541b7a8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/cfb9ecc59cf84718a97058e9c541b7a8 2024-11-16T05:40:42,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/cfb9ecc59cf84718a97058e9c541b7a8, entries=150, sequenceid=193, filesize=11.9 K 2024-11-16T05:40:42,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e8755229bb6a5d7b53307633970aa7aa in 498ms, sequenceid=193, compaction requested=false 2024-11-16T05:40:42,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:43,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:40:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:43,018 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T05:40:43,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/8e9b9af1195d44d28ef865b14d5bcf63 is 50, key is test_row_0/A:col10/1731735642390/Put/seqid=0 2024-11-16T05:40:43,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742123_1299 (size=12151) 2024-11-16T05:40:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:43,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:40:43,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735703093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735703096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735703097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735703098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735703099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735703208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735703208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735703208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735703208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735703209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735703419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735703419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735703419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735703419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735703419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,446 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/8e9b9af1195d44d28ef865b14d5bcf63 2024-11-16T05:40:43,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/1a44054bc88c41d08c182d04f18b4415 is 50, key is test_row_0/B:col10/1731735642390/Put/seqid=0 2024-11-16T05:40:43,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742124_1300 (size=12151) 2024-11-16T05:40:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735703728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735703728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735703728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735703728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735703728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:43,920 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/1a44054bc88c41d08c182d04f18b4415 2024-11-16T05:40:43,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/00bdfa16ddb24d26a85a4a6aa28b86e8 is 50, key is test_row_0/C:col10/1731735642390/Put/seqid=0 2024-11-16T05:40:43,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742125_1301 (size=12151) 2024-11-16T05:40:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:40:44,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735704237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:44,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735704238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:44,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735704238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:44,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735704238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:44,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735704238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:44,342 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/00bdfa16ddb24d26a85a4a6aa28b86e8 2024-11-16T05:40:44,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/8e9b9af1195d44d28ef865b14d5bcf63 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8e9b9af1195d44d28ef865b14d5bcf63 2024-11-16T05:40:44,363 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8e9b9af1195d44d28ef865b14d5bcf63, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T05:40:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/1a44054bc88c41d08c182d04f18b4415 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1a44054bc88c41d08c182d04f18b4415 2024-11-16T05:40:44,370 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1a44054bc88c41d08c182d04f18b4415, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T05:40:44,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/00bdfa16ddb24d26a85a4a6aa28b86e8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/00bdfa16ddb24d26a85a4a6aa28b86e8 2024-11-16T05:40:44,384 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/00bdfa16ddb24d26a85a4a6aa28b86e8, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T05:40:44,385 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e8755229bb6a5d7b53307633970aa7aa in 1367ms, sequenceid=208, compaction requested=true 2024-11-16T05:40:44,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:44,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:44,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-16T05:40:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-16T05:40:44,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-16T05:40:44,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4450 sec 2024-11-16T05:40:44,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.4510 sec 2024-11-16T05:40:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:45,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T05:40:45,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/7f4f949732f94079abe46f52d572b3d6 is 50, key is test_row_0/A:col10/1731735643095/Put/seqid=0 2024-11-16T05:40:45,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742126_1302 (size=12151) 2024-11-16T05:40:45,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/7f4f949732f94079abe46f52d572b3d6 2024-11-16T05:40:45,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735705266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735705267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735705269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735705269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735705274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/64f546b562354664bee325e399b4a8c2 is 50, key is test_row_0/B:col10/1731735643095/Put/seqid=0 2024-11-16T05:40:45,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742127_1303 (size=12151) 2024-11-16T05:40:45,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735705378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735705378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735705378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735705378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735705388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735705588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735705588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735705588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735705589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735705599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/64f546b562354664bee325e399b4a8c2 2024-11-16T05:40:45,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/d1d89f0b5bb74806b8985e74e8c46064 is 50, key is test_row_0/C:col10/1731735643095/Put/seqid=0 2024-11-16T05:40:45,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742128_1304 (size=12151) 2024-11-16T05:40:45,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/d1d89f0b5bb74806b8985e74e8c46064 2024-11-16T05:40:45,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/7f4f949732f94079abe46f52d572b3d6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/7f4f949732f94079abe46f52d572b3d6 2024-11-16T05:40:45,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/7f4f949732f94079abe46f52d572b3d6, entries=150, sequenceid=232, filesize=11.9 K 2024-11-16T05:40:45,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/64f546b562354664bee325e399b4a8c2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/64f546b562354664bee325e399b4a8c2 2024-11-16T05:40:45,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/64f546b562354664bee325e399b4a8c2, entries=150, sequenceid=232, filesize=11.9 K 2024-11-16T05:40:45,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/d1d89f0b5bb74806b8985e74e8c46064 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d1d89f0b5bb74806b8985e74e8c46064 2024-11-16T05:40:45,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d1d89f0b5bb74806b8985e74e8c46064, entries=150, sequenceid=232, filesize=11.9 K 2024-11-16T05:40:45,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e8755229bb6a5d7b53307633970aa7aa in 519ms, sequenceid=232, compaction requested=true 2024-11-16T05:40:45,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:45,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:45,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:45,771 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:45,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:45,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:45,771 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:45,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:45,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:45,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:45,774 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:45,774 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/A is initiating minor compaction (all files) 2024-11-16T05:40:45,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/B is initiating minor compaction (all files) 2024-11-16T05:40:45,774 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/B in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:45,774 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/A in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:45,774 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/5c9804378e4c4980a1322223e88299b6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b45b2f04728a41de86cb37b71cfe4efd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8e9b9af1195d44d28ef865b14d5bcf63, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/7f4f949732f94079abe46f52d572b3d6] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=50.2 K 2024-11-16T05:40:45,774 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3e7a91a270f34fb6b29e5a9cfd7e0ed1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/002ee3f8edf44be99438a604476b9bf9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1a44054bc88c41d08c182d04f18b4415, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/64f546b562354664bee325e399b4a8c2] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=47.9 K 2024-11-16T05:40:45,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e7a91a270f34fb6b29e5a9cfd7e0ed1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731735641052 2024-11-16T05:40:45,774 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c9804378e4c4980a1322223e88299b6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731735641052 2024-11-16T05:40:45,775 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 002ee3f8edf44be99438a604476b9bf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731735641738 2024-11-16T05:40:45,775 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b45b2f04728a41de86cb37b71cfe4efd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731735641738 2024-11-16T05:40:45,775 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9b9af1195d44d28ef865b14d5bcf63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731735642390 2024-11-16T05:40:45,775 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a44054bc88c41d08c182d04f18b4415, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731735642390 2024-11-16T05:40:45,776 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4f949732f94079abe46f52d572b3d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731735643095 2024-11-16T05:40:45,776 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 64f546b562354664bee325e399b4a8c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731735643095 2024-11-16T05:40:45,788 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#A#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:45,788 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/8214ce84cfd740ba8a888562765ae2e2 is 50, key is test_row_0/A:col10/1731735643095/Put/seqid=0 2024-11-16T05:40:45,799 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:45,800 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/aa44576c9df54483abd548154341d0d9 is 50, key is test_row_0/B:col10/1731735643095/Put/seqid=0 2024-11-16T05:40:45,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742129_1305 (size=12697) 2024-11-16T05:40:45,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742130_1306 (size=12697) 2024-11-16T05:40:45,821 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/aa44576c9df54483abd548154341d0d9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/aa44576c9df54483abd548154341d0d9 2024-11-16T05:40:45,827 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/B of e8755229bb6a5d7b53307633970aa7aa into aa44576c9df54483abd548154341d0d9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:45,827 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:45,827 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/B, priority=12, startTime=1731735645771; duration=0sec 2024-11-16T05:40:45,827 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:45,827 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:B 2024-11-16T05:40:45,827 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:45,828 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:45,828 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/C is initiating minor compaction (all files) 2024-11-16T05:40:45,828 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/C in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:45,828 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3ade528461374f70b024cad06ef4d2da, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/cfb9ecc59cf84718a97058e9c541b7a8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/00bdfa16ddb24d26a85a4a6aa28b86e8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d1d89f0b5bb74806b8985e74e8c46064] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=47.8 K 2024-11-16T05:40:45,829 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ade528461374f70b024cad06ef4d2da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731735641052 2024-11-16T05:40:45,829 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb9ecc59cf84718a97058e9c541b7a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731735641738 2024-11-16T05:40:45,829 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 00bdfa16ddb24d26a85a4a6aa28b86e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731735642390 2024-11-16T05:40:45,829 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting d1d89f0b5bb74806b8985e74e8c46064, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731735643095 2024-11-16T05:40:45,836 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#C#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:45,836 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/393bd21acad9468ca46dd1d754eb58a3 is 50, key is test_row_0/C:col10/1731735643095/Put/seqid=0 2024-11-16T05:40:45,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742131_1307 (size=12663) 2024-11-16T05:40:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:45,900 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:45,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T05:40:45,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/4f09e2b735be47be9a45e77cf0201195 is 50, key is test_row_0/A:col10/1731735645900/Put/seqid=0 2024-11-16T05:40:45,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742132_1308 (size=12151) 2024-11-16T05:40:45,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735705918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735705921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735705922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735705922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:45,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735705922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:45,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735706028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735706029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735706029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735706029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735706029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:40:46,099 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:46,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-16T05:40:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:40:46,101 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:46,101 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:46,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:46,208 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/8214ce84cfd740ba8a888562765ae2e2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8214ce84cfd740ba8a888562765ae2e2 2024-11-16T05:40:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:40:46,212 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/A of e8755229bb6a5d7b53307633970aa7aa into 8214ce84cfd740ba8a888562765ae2e2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:46,212 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:46,212 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/A, priority=12, startTime=1731735645771; duration=0sec 2024-11-16T05:40:46,213 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:46,213 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:A 2024-11-16T05:40:46,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735706238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735706238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735706238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735706238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735706238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,246 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/393bd21acad9468ca46dd1d754eb58a3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/393bd21acad9468ca46dd1d754eb58a3 2024-11-16T05:40:46,250 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/C of e8755229bb6a5d7b53307633970aa7aa into 393bd21acad9468ca46dd1d754eb58a3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:46,250 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:46,251 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/C, priority=12, startTime=1731735645771; duration=0sec 2024-11-16T05:40:46,251 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:46,251 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:C 2024-11-16T05:40:46,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T05:40:46,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:46,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:46,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:46,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/4f09e2b735be47be9a45e77cf0201195 2024-11-16T05:40:46,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/a0dfede4c9ee4bd185617e35a3a86f15 is 50, key is test_row_0/B:col10/1731735645900/Put/seqid=0 2024-11-16T05:40:46,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742133_1309 (size=12151) 2024-11-16T05:40:46,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T05:40:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:46,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:40:46,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735706548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735706548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735706548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735706548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735706548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:46,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T05:40:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:46,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T05:40:46,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:46,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:46,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:46,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:40:46,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/a0dfede4c9ee4bd185617e35a3a86f15 2024-11-16T05:40:46,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/ba1c5e99911345478c56269df53c1b2c is 50, key is test_row_0/C:col10/1731735645900/Put/seqid=0 2024-11-16T05:40:46,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742134_1310 (size=12151) 2024-11-16T05:40:46,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T05:40:46,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:46,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:46,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:46,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:47,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T05:40:47,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:47,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:47,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:47,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:47,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:47,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735707058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:47,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735707058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:47,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735707058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:47,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735707059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:47,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735707059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:47,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/ba1c5e99911345478c56269df53c1b2c 2024-11-16T05:40:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/4f09e2b735be47be9a45e77cf0201195 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/4f09e2b735be47be9a45e77cf0201195 2024-11-16T05:40:47,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/4f09e2b735be47be9a45e77cf0201195, entries=150, sequenceid=249, filesize=11.9 K 2024-11-16T05:40:47,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/a0dfede4c9ee4bd185617e35a3a86f15 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a0dfede4c9ee4bd185617e35a3a86f15 2024-11-16T05:40:47,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a0dfede4c9ee4bd185617e35a3a86f15, entries=150, sequenceid=249, filesize=11.9 K 2024-11-16T05:40:47,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T05:40:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:47,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:47,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/ba1c5e99911345478c56269df53c1b2c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ba1c5e99911345478c56269df53c1b2c 2024-11-16T05:40:47,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ba1c5e99911345478c56269df53c1b2c, entries=150, sequenceid=249, filesize=11.9 K 2024-11-16T05:40:47,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for e8755229bb6a5d7b53307633970aa7aa in 1286ms, sequenceid=249, compaction requested=false 2024-11-16T05:40:47,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:40:47,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T05:40:47,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:47,327 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T05:40:47,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/8dc451065e484fc58034dd1832e29cc6 is 50, key is test_row_0/A:col10/1731735645921/Put/seqid=0 2024-11-16T05:40:47,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742135_1311 (size=12301) 2024-11-16T05:40:47,760 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/8dc451065e484fc58034dd1832e29cc6 2024-11-16T05:40:47,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/cc7d0eccdc2e4841b38450d31c4e5475 is 50, key is test_row_0/B:col10/1731735645921/Put/seqid=0 2024-11-16T05:40:47,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742136_1312 (size=12301) 2024-11-16T05:40:47,772 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/cc7d0eccdc2e4841b38450d31c4e5475 2024-11-16T05:40:47,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/e1334497d44c451aab317b656ab09ce3 is 50, key is test_row_0/C:col10/1731735645921/Put/seqid=0 2024-11-16T05:40:47,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742137_1313 (size=12301) 2024-11-16T05:40:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:48,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:48,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735708080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735708082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735708082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735708083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735708084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,187 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/e1334497d44c451aab317b656ab09ce3 2024-11-16T05:40:48,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735708188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735708188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735708188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/8dc451065e484fc58034dd1832e29cc6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8dc451065e484fc58034dd1832e29cc6 2024-11-16T05:40:48,195 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8dc451065e484fc58034dd1832e29cc6, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T05:40:48,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/cc7d0eccdc2e4841b38450d31c4e5475 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/cc7d0eccdc2e4841b38450d31c4e5475 2024-11-16T05:40:48,199 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/cc7d0eccdc2e4841b38450d31c4e5475, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T05:40:48,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735708198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/e1334497d44c451aab317b656ab09ce3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e1334497d44c451aab317b656ab09ce3 2024-11-16T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735708199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,204 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e1334497d44c451aab317b656ab09ce3, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T05:40:48,204 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for e8755229bb6a5d7b53307633970aa7aa in 877ms, sequenceid=272, compaction requested=true 2024-11-16T05:40:48,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:48,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:48,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-16T05:40:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-16T05:40:48,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-16T05:40:48,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1040 sec 2024-11-16T05:40:48,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1090 sec 2024-11-16T05:40:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:40:48,258 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:48,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-16T05:40:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T05:40:48,260 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:48,261 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:48,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T05:40:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:48,401 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:48,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T05:40:48,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/c044b30cea884624a07d3fb3f9f0bf94 is 50, key is test_row_0/A:col10/1731735648401/Put/seqid=0 2024-11-16T05:40:48,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T05:40:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:48,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735708421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735708425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735708425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735708426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735708427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742138_1314 (size=17181) 2024-11-16T05:40:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735708538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735708548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735708549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735708549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735708549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T05:40:48,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:48,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:48,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:48,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T05:40:48,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T05:40:48,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:48,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:48,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:48,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735708748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735708758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735708758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735708758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735708758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:48,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/c044b30cea884624a07d3fb3f9f0bf94 2024-11-16T05:40:48,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/44701206dbce4402985cdb6d31a0554c is 50, key is test_row_0/B:col10/1731735648401/Put/seqid=0 2024-11-16T05:40:48,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742139_1315 (size=12301) 2024-11-16T05:40:48,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T05:40:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:48,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T05:40:49,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T05:40:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:49,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735709058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735709068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735709069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735709069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735709069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T05:40:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:49,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:49,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/44701206dbce4402985cdb6d31a0554c 2024-11-16T05:40:49,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/385b96833e4c4aa7b8871a6cbb29bf62 is 50, key is test_row_0/C:col10/1731735648401/Put/seqid=0 2024-11-16T05:40:49,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742140_1316 (size=12301) 2024-11-16T05:40:49,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/385b96833e4c4aa7b8871a6cbb29bf62 2024-11-16T05:40:49,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/c044b30cea884624a07d3fb3f9f0bf94 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c044b30cea884624a07d3fb3f9f0bf94 2024-11-16T05:40:49,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c044b30cea884624a07d3fb3f9f0bf94, entries=250, sequenceid=289, filesize=16.8 K 2024-11-16T05:40:49,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/44701206dbce4402985cdb6d31a0554c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/44701206dbce4402985cdb6d31a0554c 2024-11-16T05:40:49,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/44701206dbce4402985cdb6d31a0554c, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T05:40:49,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/385b96833e4c4aa7b8871a6cbb29bf62 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/385b96833e4c4aa7b8871a6cbb29bf62 2024-11-16T05:40:49,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/385b96833e4c4aa7b8871a6cbb29bf62, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T05:40:49,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e8755229bb6a5d7b53307633970aa7aa in 920ms, sequenceid=289, compaction requested=true 2024-11-16T05:40:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:49,322 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:49,322 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:49,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:49,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:49,323 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54330 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:49,323 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/A is initiating minor compaction (all files) 2024-11-16T05:40:49,323 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:49,323 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/B is initiating minor compaction (all files) 2024-11-16T05:40:49,323 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/A in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:49,323 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/B in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:49,323 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8214ce84cfd740ba8a888562765ae2e2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/4f09e2b735be47be9a45e77cf0201195, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8dc451065e484fc58034dd1832e29cc6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c044b30cea884624a07d3fb3f9f0bf94] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=53.1 K 2024-11-16T05:40:49,323 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/aa44576c9df54483abd548154341d0d9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a0dfede4c9ee4bd185617e35a3a86f15, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/cc7d0eccdc2e4841b38450d31c4e5475, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/44701206dbce4402985cdb6d31a0554c] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=48.3 K 2024-11-16T05:40:49,324 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting aa44576c9df54483abd548154341d0d9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731735643095 2024-11-16T05:40:49,324 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8214ce84cfd740ba8a888562765ae2e2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731735643095 2024-11-16T05:40:49,324 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f09e2b735be47be9a45e77cf0201195, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731735645899 2024-11-16T05:40:49,324 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting a0dfede4c9ee4bd185617e35a3a86f15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731735645899 2024-11-16T05:40:49,324 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting cc7d0eccdc2e4841b38450d31c4e5475, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731735645916 2024-11-16T05:40:49,324 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dc451065e484fc58034dd1832e29cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731735645916 2024-11-16T05:40:49,325 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 44701206dbce4402985cdb6d31a0554c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731735648080 2024-11-16T05:40:49,325 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c044b30cea884624a07d3fb3f9f0bf94, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731735648070 2024-11-16T05:40:49,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T05:40:49,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:49,332 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T05:40:49,335 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#B#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:49,336 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/ec0af248500340fab0da3e191e85102f is 50, key is test_row_0/B:col10/1731735648401/Put/seqid=0 2024-11-16T05:40:49,337 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:49,338 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/c6d6d59fa0724e029b6f9c270e6adef2 is 50, key is test_row_0/A:col10/1731735648401/Put/seqid=0 2024-11-16T05:40:49,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/f9377c6ebae94290b7b6f18561e85497 is 50, key is test_row_0/A:col10/1731735648423/Put/seqid=0 2024-11-16T05:40:49,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742141_1317 (size=12983) 2024-11-16T05:40:49,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742143_1319 (size=12983) 2024-11-16T05:40:49,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742142_1318 (size=12301) 2024-11-16T05:40:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T05:40:49,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:49,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735709590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735709592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735709593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735709597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735709598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735709708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735709708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735709708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735709708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735709708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,756 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/ec0af248500340fab0da3e191e85102f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ec0af248500340fab0da3e191e85102f 2024-11-16T05:40:49,760 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/B of e8755229bb6a5d7b53307633970aa7aa into ec0af248500340fab0da3e191e85102f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:49,760 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:49,760 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/B, priority=12, startTime=1731735649322; duration=0sec 2024-11-16T05:40:49,760 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:49,760 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:B 2024-11-16T05:40:49,760 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:40:49,762 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:40:49,762 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/C is initiating minor compaction (all files) 2024-11-16T05:40:49,762 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/C in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:49,762 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/393bd21acad9468ca46dd1d754eb58a3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ba1c5e99911345478c56269df53c1b2c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e1334497d44c451aab317b656ab09ce3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/385b96833e4c4aa7b8871a6cbb29bf62] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=48.3 K 2024-11-16T05:40:49,763 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 393bd21acad9468ca46dd1d754eb58a3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731735643095 2024-11-16T05:40:49,763 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ba1c5e99911345478c56269df53c1b2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731735645899 2024-11-16T05:40:49,763 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e1334497d44c451aab317b656ab09ce3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731735645916 2024-11-16T05:40:49,763 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 385b96833e4c4aa7b8871a6cbb29bf62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731735648080 2024-11-16T05:40:49,771 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:49,771 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/47fd85d11ae64dbaabc3667703fff5ee is 50, key is test_row_0/C:col10/1731735648401/Put/seqid=0 2024-11-16T05:40:49,786 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/c6d6d59fa0724e029b6f9c270e6adef2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c6d6d59fa0724e029b6f9c270e6adef2 2024-11-16T05:40:49,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742144_1320 (size=12949) 2024-11-16T05:40:49,793 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/A of e8755229bb6a5d7b53307633970aa7aa into c6d6d59fa0724e029b6f9c270e6adef2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:49,793 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:49,793 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/A, priority=12, startTime=1731735649322; duration=0sec 2024-11-16T05:40:49,793 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:49,793 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:A 2024-11-16T05:40:49,795 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/f9377c6ebae94290b7b6f18561e85497 2024-11-16T05:40:49,795 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/47fd85d11ae64dbaabc3667703fff5ee as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/47fd85d11ae64dbaabc3667703fff5ee 2024-11-16T05:40:49,801 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/C of e8755229bb6a5d7b53307633970aa7aa into 47fd85d11ae64dbaabc3667703fff5ee(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:49,801 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:49,801 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/C, priority=12, startTime=1731735649322; duration=0sec 2024-11-16T05:40:49,801 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:49,801 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:C 2024-11-16T05:40:49,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/849c072bef8a4db4a4addbbd54243bdc is 50, key is test_row_0/B:col10/1731735648423/Put/seqid=0 2024-11-16T05:40:49,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742145_1321 (size=12301) 2024-11-16T05:40:49,808 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/849c072bef8a4db4a4addbbd54243bdc 2024-11-16T05:40:49,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/4777508fece94de691f510a69d2f5124 is 50, key is test_row_0/C:col10/1731735648423/Put/seqid=0 2024-11-16T05:40:49,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742146_1322 (size=12301) 2024-11-16T05:40:49,849 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/4777508fece94de691f510a69d2f5124 2024-11-16T05:40:49,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/f9377c6ebae94290b7b6f18561e85497 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9377c6ebae94290b7b6f18561e85497 2024-11-16T05:40:49,862 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9377c6ebae94290b7b6f18561e85497, entries=150, sequenceid=308, filesize=12.0 K 2024-11-16T05:40:49,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/849c072bef8a4db4a4addbbd54243bdc as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/849c072bef8a4db4a4addbbd54243bdc 2024-11-16T05:40:49,871 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/849c072bef8a4db4a4addbbd54243bdc, entries=150, sequenceid=308, filesize=12.0 K 2024-11-16T05:40:49,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/4777508fece94de691f510a69d2f5124 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/4777508fece94de691f510a69d2f5124 2024-11-16T05:40:49,877 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/4777508fece94de691f510a69d2f5124, entries=150, sequenceid=308, filesize=12.0 K 2024-11-16T05:40:49,878 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for e8755229bb6a5d7b53307633970aa7aa in 546ms, sequenceid=308, compaction requested=false 2024-11-16T05:40:49,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:49,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:49,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-16T05:40:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-16T05:40:49,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-16T05:40:49,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6190 sec 2024-11-16T05:40:49,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.6250 sec 2024-11-16T05:40:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:49,920 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:49,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T05:40:49,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/f557752b6a804cea9251f8719801c0f1 is 50, key is test_row_0/A:col10/1731735649596/Put/seqid=0 2024-11-16T05:40:49,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742147_1323 (size=12301) 2024-11-16T05:40:49,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735709941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735709942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735709945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735709946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:49,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735709948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735710058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735710058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735710058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735710058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735710058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735710268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735710268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735710269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735710269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735710271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/f557752b6a804cea9251f8719801c0f1 2024-11-16T05:40:50,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/09a9d6d08e6b4aeb97e73810bc90e27b is 50, key is test_row_0/B:col10/1731735649596/Put/seqid=0 2024-11-16T05:40:50,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742148_1324 (size=12301) 2024-11-16T05:40:50,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/09a9d6d08e6b4aeb97e73810bc90e27b 2024-11-16T05:40:50,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/1130dfb219054bd09f3417062b16771c is 50, key is test_row_0/C:col10/1731735649596/Put/seqid=0 2024-11-16T05:40:50,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T05:40:50,418 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:50,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-16T05:40:50,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T05:40:50,421 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:50,422 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:50,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:50,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742149_1325 (size=12301) 2024-11-16T05:40:50,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/1130dfb219054bd09f3417062b16771c 2024-11-16T05:40:50,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/f557752b6a804cea9251f8719801c0f1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f557752b6a804cea9251f8719801c0f1 2024-11-16T05:40:50,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f557752b6a804cea9251f8719801c0f1, entries=150, sequenceid=329, filesize=12.0 K 2024-11-16T05:40:50,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/09a9d6d08e6b4aeb97e73810bc90e27b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/09a9d6d08e6b4aeb97e73810bc90e27b 2024-11-16T05:40:50,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/09a9d6d08e6b4aeb97e73810bc90e27b, entries=150, sequenceid=329, filesize=12.0 K 2024-11-16T05:40:50,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/1130dfb219054bd09f3417062b16771c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1130dfb219054bd09f3417062b16771c 2024-11-16T05:40:50,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1130dfb219054bd09f3417062b16771c, entries=150, sequenceid=329, filesize=12.0 K 2024-11-16T05:40:50,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e8755229bb6a5d7b53307633970aa7aa in 528ms, sequenceid=329, compaction requested=true 2024-11-16T05:40:50,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:50,449 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:50,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:50,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:50,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:50,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:50,449 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:50,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:50,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:50,450 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:50,450 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/B is initiating minor compaction (all files) 2024-11-16T05:40:50,450 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/B in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:50,451 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ec0af248500340fab0da3e191e85102f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/849c072bef8a4db4a4addbbd54243bdc, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/09a9d6d08e6b4aeb97e73810bc90e27b] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=36.7 K 2024-11-16T05:40:50,451 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:50,451 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ec0af248500340fab0da3e191e85102f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731735648080 2024-11-16T05:40:50,451 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/A is initiating minor compaction (all files) 2024-11-16T05:40:50,451 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/A in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:50,451 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c6d6d59fa0724e029b6f9c270e6adef2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9377c6ebae94290b7b6f18561e85497, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f557752b6a804cea9251f8719801c0f1] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=36.7 K 2024-11-16T05:40:50,455 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 849c072bef8a4db4a4addbbd54243bdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731735648423 2024-11-16T05:40:50,455 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d6d59fa0724e029b6f9c270e6adef2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731735648080 2024-11-16T05:40:50,456 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 09a9d6d08e6b4aeb97e73810bc90e27b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731735649595 2024-11-16T05:40:50,456 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9377c6ebae94290b7b6f18561e85497, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731735648423 2024-11-16T05:40:50,456 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f557752b6a804cea9251f8719801c0f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731735649595 2024-11-16T05:40:50,467 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#B#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:50,468 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/b90bc329db4e48dab288076e946ab0ec is 50, key is test_row_0/B:col10/1731735649596/Put/seqid=0 2024-11-16T05:40:50,475 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:50,476 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/6bc12925e476409195bcb4b89bd4d27d is 50, key is test_row_0/A:col10/1731735649596/Put/seqid=0 2024-11-16T05:40:50,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742150_1326 (size=13085) 2024-11-16T05:40:50,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742151_1327 (size=13085) 2024-11-16T05:40:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T05:40:50,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T05:40:50,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:50,575 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T05:40:50,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/c570fbf244224b0a9cee3e237feaf076 is 50, key is test_row_0/A:col10/1731735649945/Put/seqid=0 2024-11-16T05:40:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:50,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:50,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742152_1328 (size=12301) 2024-11-16T05:40:50,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735710590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735710591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735710593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735710594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735710594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735710698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735710698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735710708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735710708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735710708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T05:40:50,898 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/b90bc329db4e48dab288076e946ab0ec as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b90bc329db4e48dab288076e946ab0ec 2024-11-16T05:40:50,901 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/6bc12925e476409195bcb4b89bd4d27d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/6bc12925e476409195bcb4b89bd4d27d 2024-11-16T05:40:50,903 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/B of e8755229bb6a5d7b53307633970aa7aa into b90bc329db4e48dab288076e946ab0ec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:50,903 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:50,903 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/B, priority=13, startTime=1731735650449; duration=0sec 2024-11-16T05:40:50,903 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:50,903 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:B 2024-11-16T05:40:50,903 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:40:50,904 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:40:50,904 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/C is initiating minor compaction (all files) 2024-11-16T05:40:50,904 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/C in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:50,904 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/47fd85d11ae64dbaabc3667703fff5ee, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/4777508fece94de691f510a69d2f5124, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1130dfb219054bd09f3417062b16771c] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=36.7 K 2024-11-16T05:40:50,904 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 47fd85d11ae64dbaabc3667703fff5ee, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731735648080 2024-11-16T05:40:50,905 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4777508fece94de691f510a69d2f5124, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731735648423 2024-11-16T05:40:50,905 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1130dfb219054bd09f3417062b16771c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731735649595 2024-11-16T05:40:50,905 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/A of e8755229bb6a5d7b53307633970aa7aa into 6bc12925e476409195bcb4b89bd4d27d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:50,905 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:50,905 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/A, priority=13, startTime=1731735650449; duration=0sec 2024-11-16T05:40:50,906 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:50,906 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:A 2024-11-16T05:40:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735710908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735710908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,912 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#C#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:50,913 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/46661510f1034c4baa669764f8285b30 is 50, key is test_row_0/C:col10/1731735649596/Put/seqid=0 2024-11-16T05:40:50,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742153_1329 (size=13051) 2024-11-16T05:40:50,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735710918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,921 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/46661510f1034c4baa669764f8285b30 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/46661510f1034c4baa669764f8285b30 2024-11-16T05:40:50,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735710929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,930 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/C of e8755229bb6a5d7b53307633970aa7aa into 46661510f1034c4baa669764f8285b30(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:50,930 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:50,930 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/C, priority=13, startTime=1731735650449; duration=0sec 2024-11-16T05:40:50,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735710929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:50,930 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:50,930 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:C 2024-11-16T05:40:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:50,984 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/c570fbf244224b0a9cee3e237feaf076 2024-11-16T05:40:50,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/181a722da3ce453ab4ee5bd3db650934 is 50, key is test_row_0/B:col10/1731735649945/Put/seqid=0 2024-11-16T05:40:50,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742154_1330 (size=12301) 2024-11-16T05:40:50,995 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/181a722da3ce453ab4ee5bd3db650934 2024-11-16T05:40:51,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/e7012606f2d742c6b7adb0dc2be25cb1 is 50, key is test_row_0/C:col10/1731735649945/Put/seqid=0 2024-11-16T05:40:51,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742155_1331 (size=12301) 2024-11-16T05:40:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T05:40:51,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735711218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735711218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735711228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735711238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735711238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,406 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/e7012606f2d742c6b7adb0dc2be25cb1 2024-11-16T05:40:51,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/c570fbf244224b0a9cee3e237feaf076 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c570fbf244224b0a9cee3e237feaf076 2024-11-16T05:40:51,413 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c570fbf244224b0a9cee3e237feaf076, entries=150, sequenceid=347, filesize=12.0 K 2024-11-16T05:40:51,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/181a722da3ce453ab4ee5bd3db650934 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/181a722da3ce453ab4ee5bd3db650934 2024-11-16T05:40:51,418 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/181a722da3ce453ab4ee5bd3db650934, entries=150, sequenceid=347, filesize=12.0 K 2024-11-16T05:40:51,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/e7012606f2d742c6b7adb0dc2be25cb1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e7012606f2d742c6b7adb0dc2be25cb1 2024-11-16T05:40:51,422 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e7012606f2d742c6b7adb0dc2be25cb1, entries=150, sequenceid=347, filesize=12.0 K 2024-11-16T05:40:51,423 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e8755229bb6a5d7b53307633970aa7aa in 848ms, sequenceid=347, compaction requested=false 2024-11-16T05:40:51,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:51,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:51,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-16T05:40:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-16T05:40:51,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-16T05:40:51,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0030 sec 2024-11-16T05:40:51,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.0070 sec 2024-11-16T05:40:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T05:40:51,559 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:51,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-16T05:40:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T05:40:51,561 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:51,562 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:51,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T05:40:51,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T05:40:51,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:51,716 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T05:40:51,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/406e3f55a7d74eab89b1815eb69bac64 is 50, key is test_row_0/A:col10/1731735650592/Put/seqid=0 2024-11-16T05:40:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:51,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:51,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742156_1332 (size=12301) 2024-11-16T05:40:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735711758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735711759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735711757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735711764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735711764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735711868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735711868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735711868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T05:40:51,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735711878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735711878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735712078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735712078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735712078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735712088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735712088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,148 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/406e3f55a7d74eab89b1815eb69bac64 2024-11-16T05:40:52,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/96c75296f06c433f9ddeb3499839d6cb is 50, key is test_row_0/B:col10/1731735650592/Put/seqid=0 2024-11-16T05:40:52,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742157_1333 (size=12301) 2024-11-16T05:40:52,158 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/96c75296f06c433f9ddeb3499839d6cb 2024-11-16T05:40:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/28e8ba265f38472ca3b5697f1020a52f is 50, key is test_row_0/C:col10/1731735650592/Put/seqid=0 2024-11-16T05:40:52,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742158_1334 (size=12301) 2024-11-16T05:40:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T05:40:52,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735712388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735712389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735712389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735712398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735712398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,568 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/28e8ba265f38472ca3b5697f1020a52f 2024-11-16T05:40:52,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/406e3f55a7d74eab89b1815eb69bac64 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/406e3f55a7d74eab89b1815eb69bac64 2024-11-16T05:40:52,576 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/406e3f55a7d74eab89b1815eb69bac64, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T05:40:52,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/96c75296f06c433f9ddeb3499839d6cb as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/96c75296f06c433f9ddeb3499839d6cb 2024-11-16T05:40:52,581 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/96c75296f06c433f9ddeb3499839d6cb, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T05:40:52,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/28e8ba265f38472ca3b5697f1020a52f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/28e8ba265f38472ca3b5697f1020a52f 2024-11-16T05:40:52,586 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/28e8ba265f38472ca3b5697f1020a52f, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T05:40:52,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e8755229bb6a5d7b53307633970aa7aa in 871ms, sequenceid=368, compaction requested=true 2024-11-16T05:40:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-16T05:40:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-16T05:40:52,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-16T05:40:52,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0270 sec 2024-11-16T05:40:52,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0340 sec 2024-11-16T05:40:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T05:40:52,698 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:52,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-16T05:40:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T05:40:52,701 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:52,702 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:52,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T05:40:52,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-16T05:40:52,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:52,855 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T05:40:52,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/80186cfbc0a648728f8e1e5344f9c69c is 50, key is test_row_0/A:col10/1731735651756/Put/seqid=0 2024-11-16T05:40:52,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742159_1335 (size=12301) 2024-11-16T05:40:52,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:52,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735712955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735712953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735712955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735712956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:52,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735712959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:52,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:52,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T05:40:53,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735713068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735713068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735713068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735713068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735713078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,266 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/80186cfbc0a648728f8e1e5344f9c69c 2024-11-16T05:40:53,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/a6b0119a7ffa46bca5b53ad99b266e1b is 50, key is test_row_0/B:col10/1731735651756/Put/seqid=0 2024-11-16T05:40:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735713278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735713278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735713278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735713278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735713288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742160_1336 (size=12301) 2024-11-16T05:40:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T05:40:53,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735713588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735713588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735713589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735713589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735713598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:53,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:53,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:53,703 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/a6b0119a7ffa46bca5b53ad99b266e1b 2024-11-16T05:40:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/f75f39a2dfa74c9bb29321ff08e282c2 is 50, key is test_row_0/C:col10/1731735651756/Put/seqid=0 2024-11-16T05:40:53,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742161_1337 (size=12301) 2024-11-16T05:40:53,751 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/f75f39a2dfa74c9bb29321ff08e282c2 2024-11-16T05:40:53,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/80186cfbc0a648728f8e1e5344f9c69c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/80186cfbc0a648728f8e1e5344f9c69c 2024-11-16T05:40:53,763 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/80186cfbc0a648728f8e1e5344f9c69c, entries=150, sequenceid=386, filesize=12.0 K 2024-11-16T05:40:53,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/a6b0119a7ffa46bca5b53ad99b266e1b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a6b0119a7ffa46bca5b53ad99b266e1b 2024-11-16T05:40:53,776 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a6b0119a7ffa46bca5b53ad99b266e1b, entries=150, sequenceid=386, filesize=12.0 K 2024-11-16T05:40:53,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/f75f39a2dfa74c9bb29321ff08e282c2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/f75f39a2dfa74c9bb29321ff08e282c2 2024-11-16T05:40:53,790 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/f75f39a2dfa74c9bb29321ff08e282c2, entries=150, sequenceid=386, filesize=12.0 K 2024-11-16T05:40:53,791 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e8755229bb6a5d7b53307633970aa7aa in 936ms, sequenceid=386, compaction requested=true 2024-11-16T05:40:53,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:53,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:53,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-16T05:40:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-16T05:40:53,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-16T05:40:53,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0920 sec 2024-11-16T05:40:53,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.0980 sec 2024-11-16T05:40:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T05:40:53,839 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:53,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-16T05:40:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T05:40:53,843 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:53,843 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:53,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T05:40:53,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-16T05:40:53,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:53,997 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T05:40:54,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/dd757944c9024cc791e24b74b51c8a27 is 50, key is test_row_0/A:col10/1731735652958/Put/seqid=0 2024-11-16T05:40:54,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742162_1338 (size=12301) 2024-11-16T05:40:54,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:54,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735714125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735714129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735714130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735714130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735714131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T05:40:54,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735714238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735714238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735714252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735714252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735714253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,417 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/dd757944c9024cc791e24b74b51c8a27 2024-11-16T05:40:54,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/573140ea3e7c4448a35608e3f202a3ab is 50, key is test_row_0/B:col10/1731735652958/Put/seqid=0 2024-11-16T05:40:54,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742163_1339 (size=12301) 2024-11-16T05:40:54,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735714448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735714449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T05:40:54,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735714468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735714468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735714468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735714758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735714758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735714778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735714783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735714783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:54,849 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/573140ea3e7c4448a35608e3f202a3ab 2024-11-16T05:40:54,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/3c3e50a2fb1a47cfb491cc2c3f126273 is 50, key is test_row_0/C:col10/1731735652958/Put/seqid=0 2024-11-16T05:40:54,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742164_1340 (size=12301) 2024-11-16T05:40:54,868 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/3c3e50a2fb1a47cfb491cc2c3f126273 2024-11-16T05:40:54,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/dd757944c9024cc791e24b74b51c8a27 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/dd757944c9024cc791e24b74b51c8a27 2024-11-16T05:40:54,877 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/dd757944c9024cc791e24b74b51c8a27, entries=150, sequenceid=404, filesize=12.0 K 2024-11-16T05:40:54,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/573140ea3e7c4448a35608e3f202a3ab as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/573140ea3e7c4448a35608e3f202a3ab 2024-11-16T05:40:54,896 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/573140ea3e7c4448a35608e3f202a3ab, entries=150, sequenceid=404, filesize=12.0 K 2024-11-16T05:40:54,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/3c3e50a2fb1a47cfb491cc2c3f126273 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3c3e50a2fb1a47cfb491cc2c3f126273 2024-11-16T05:40:54,902 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3c3e50a2fb1a47cfb491cc2c3f126273, entries=150, sequenceid=404, filesize=12.0 K 2024-11-16T05:40:54,903 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for e8755229bb6a5d7b53307633970aa7aa in 906ms, sequenceid=404, compaction requested=true 2024-11-16T05:40:54,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:54,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:54,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-16T05:40:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-16T05:40:54,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-16T05:40:54,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0610 sec 2024-11-16T05:40:54,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.0680 sec 2024-11-16T05:40:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T05:40:54,978 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:54,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-16T05:40:54,982 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T05:40:54,983 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:54,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T05:40:55,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-16T05:40:55,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:55,137 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T05:40:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/72be5442763e49eb98032fc30f63b731 is 50, key is test_row_0/A:col10/1731735654129/Put/seqid=0 2024-11-16T05:40:55,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742165_1341 (size=12301) 2024-11-16T05:40:55,178 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/72be5442763e49eb98032fc30f63b731 2024-11-16T05:40:55,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/3eb99ca8a3ce4877bfe0fd91ef469ef0 is 50, key is test_row_0/B:col10/1731735654129/Put/seqid=0 2024-11-16T05:40:55,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742166_1342 (size=12301) 2024-11-16T05:40:55,227 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/3eb99ca8a3ce4877bfe0fd91ef469ef0 2024-11-16T05:40:55,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/b1009a0d90544d80bacd5a83ccb7aa58 is 50, key is test_row_0/C:col10/1731735654129/Put/seqid=0 2024-11-16T05:40:55,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742167_1343 (size=12301) 2024-11-16T05:40:55,243 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/b1009a0d90544d80bacd5a83ccb7aa58 2024-11-16T05:40:55,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/72be5442763e49eb98032fc30f63b731 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/72be5442763e49eb98032fc30f63b731 2024-11-16T05:40:55,257 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/72be5442763e49eb98032fc30f63b731, entries=150, sequenceid=423, filesize=12.0 K 2024-11-16T05:40:55,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/3eb99ca8a3ce4877bfe0fd91ef469ef0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3eb99ca8a3ce4877bfe0fd91ef469ef0 2024-11-16T05:40:55,264 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3eb99ca8a3ce4877bfe0fd91ef469ef0, entries=150, sequenceid=423, filesize=12.0 K 2024-11-16T05:40:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/b1009a0d90544d80bacd5a83ccb7aa58 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b1009a0d90544d80bacd5a83ccb7aa58 2024-11-16T05:40:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:55,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:55,272 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b1009a0d90544d80bacd5a83ccb7aa58, entries=150, sequenceid=423, filesize=12.0 K 2024-11-16T05:40:55,272 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=20.13 KB/20610 for e8755229bb6a5d7b53307633970aa7aa in 135ms, sequenceid=423, compaction requested=true 2024-11-16T05:40:55,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:55,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:55,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-16T05:40:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-16T05:40:55,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-16T05:40:55,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 291 msec 2024-11-16T05:40:55,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 298 msec 2024-11-16T05:40:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:55,283 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:40:55,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:40:55,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/f9b5ca14374b4456ac16db875736804d is 50, key is test_row_0/A:col10/1731735655270/Put/seqid=0 2024-11-16T05:40:55,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742168_1344 (size=14741) 2024-11-16T05:40:55,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/f9b5ca14374b4456ac16db875736804d 2024-11-16T05:40:55,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/b3ae2fac68d04cc39aa3d585e1c14c9f is 50, key is test_row_0/B:col10/1731735655270/Put/seqid=0 2024-11-16T05:40:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T05:40:55,305 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:55,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:40:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-16T05:40:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-16T05:40:55,308 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:40:55,308 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:40:55,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:40:55,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742169_1345 (size=12301) 2024-11-16T05:40:55,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735715351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735715352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735715353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735715355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735715356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,364 INFO [Thread-1194 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1194. 2024-11-16T05:40:55,364 DEBUG [Thread-1194 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:55,364 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:55,364 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:55,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:55,366 INFO [Thread-1198 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1198. 2024-11-16T05:40:55,366 DEBUG [Thread-1198 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:55,366 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:55,367 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:55,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:55,367 INFO [Thread-1202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1202. 2024-11-16T05:40:55,367 DEBUG [Thread-1202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:55,367 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:55,367 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:55,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:55,369 INFO [Thread-1200 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1200. 2024-11-16T05:40:55,369 DEBUG [Thread-1200 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:55,369 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:55,369 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:55,369 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:55,372 INFO [Thread-1196 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1196. 2024-11-16T05:40:55,372 DEBUG [Thread-1196 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:55,372 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:55,372 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:55,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-16T05:40:55,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-16T05:40:55,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:55,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:55,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:55,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735715468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735715468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735715468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735715468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735715468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-16T05:40:55,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:55,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:55,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:55,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-16T05:40:55,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735715679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735715679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735715679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735715679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735715679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/b3ae2fac68d04cc39aa3d585e1c14c9f 2024-11-16T05:40:55,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/2d25dc792da04c0bb37c4864dbc8b128 is 50, key is test_row_0/C:col10/1731735655270/Put/seqid=0 2024-11-16T05:40:55,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742170_1346 (size=12301) 2024-11-16T05:40:55,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-16T05:40:55,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:55,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:55,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:55,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-16T05:40:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:55,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-16T05:40:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35488 deadline: 1731735715989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35480 deadline: 1731735715990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35518 deadline: 1731735715989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35546 deadline: 1731735715990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:40:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35554 deadline: 1731735715990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:40:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:40:56,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-16T05:40:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:56,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:40:56,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/2d25dc792da04c0bb37c4864dbc8b128 2024-11-16T05:40:56,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/f9b5ca14374b4456ac16db875736804d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9b5ca14374b4456ac16db875736804d 2024-11-16T05:40:56,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9b5ca14374b4456ac16db875736804d, entries=200, sequenceid=434, filesize=14.4 K 2024-11-16T05:40:56,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/b3ae2fac68d04cc39aa3d585e1c14c9f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b3ae2fac68d04cc39aa3d585e1c14c9f 2024-11-16T05:40:56,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b3ae2fac68d04cc39aa3d585e1c14c9f, entries=150, sequenceid=434, filesize=12.0 K 2024-11-16T05:40:56,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/2d25dc792da04c0bb37c4864dbc8b128 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2d25dc792da04c0bb37c4864dbc8b128 2024-11-16T05:40:56,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2d25dc792da04c0bb37c4864dbc8b128, entries=150, sequenceid=434, filesize=12.0 K 2024-11-16T05:40:56,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e8755229bb6a5d7b53307633970aa7aa in 870ms, sequenceid=434, compaction requested=true 2024-11-16T05:40:56,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:56,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:40:56,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:56,153 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-16T05:40:56,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:40:56,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:56,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8755229bb6a5d7b53307633970aa7aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:40:56,154 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-16T05:40:56,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:56,156 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 89331 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-16T05:40:56,156 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86891 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-16T05:40:56,156 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/A is initiating minor compaction (all files) 2024-11-16T05:40:56,156 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/B is initiating minor compaction (all files) 2024-11-16T05:40:56,156 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/A in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:56,156 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/B in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:56,156 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/6bc12925e476409195bcb4b89bd4d27d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c570fbf244224b0a9cee3e237feaf076, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/406e3f55a7d74eab89b1815eb69bac64, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/80186cfbc0a648728f8e1e5344f9c69c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/dd757944c9024cc791e24b74b51c8a27, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/72be5442763e49eb98032fc30f63b731, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9b5ca14374b4456ac16db875736804d] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=87.2 K 2024-11-16T05:40:56,156 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b90bc329db4e48dab288076e946ab0ec, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/181a722da3ce453ab4ee5bd3db650934, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/96c75296f06c433f9ddeb3499839d6cb, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a6b0119a7ffa46bca5b53ad99b266e1b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/573140ea3e7c4448a35608e3f202a3ab, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3eb99ca8a3ce4877bfe0fd91ef469ef0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b3ae2fac68d04cc39aa3d585e1c14c9f] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=84.9 K 2024-11-16T05:40:56,157 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc12925e476409195bcb4b89bd4d27d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731735649595 2024-11-16T05:40:56,157 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b90bc329db4e48dab288076e946ab0ec, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731735649595 2024-11-16T05:40:56,157 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 181a722da3ce453ab4ee5bd3db650934, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731735649939 2024-11-16T05:40:56,157 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting c570fbf244224b0a9cee3e237feaf076, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731735649939 2024-11-16T05:40:56,158 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96c75296f06c433f9ddeb3499839d6cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731735650592 2024-11-16T05:40:56,158 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 406e3f55a7d74eab89b1815eb69bac64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731735650592 2024-11-16T05:40:56,158 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6b0119a7ffa46bca5b53ad99b266e1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731735651756 2024-11-16T05:40:56,158 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 80186cfbc0a648728f8e1e5344f9c69c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731735651756 2024-11-16T05:40:56,158 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 573140ea3e7c4448a35608e3f202a3ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731735652952 2024-11-16T05:40:56,158 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting dd757944c9024cc791e24b74b51c8a27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731735652952 2024-11-16T05:40:56,159 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb99ca8a3ce4877bfe0fd91ef469ef0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731735654128 2024-11-16T05:40:56,159 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 72be5442763e49eb98032fc30f63b731, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731735654128 2024-11-16T05:40:56,159 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ae2fac68d04cc39aa3d585e1c14c9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731735655270 2024-11-16T05:40:56,159 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b5ca14374b4456ac16db875736804d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731735655270 2024-11-16T05:40:56,172 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#B#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:56,172 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/414b4550257a4f0aa9cc52f1eb5b653f is 50, key is test_row_0/B:col10/1731735655270/Put/seqid=0 2024-11-16T05:40:56,173 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#A#compaction#300 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:56,174 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/0765ef70b107448c8719bf648607a7a0 is 50, key is test_row_0/A:col10/1731735655270/Put/seqid=0 2024-11-16T05:40:56,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742171_1347 (size=13323) 2024-11-16T05:40:56,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742172_1348 (size=13323) 2024-11-16T05:40:56,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-16T05:40:56,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:56,228 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T05:40:56,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/eb294ff92d424efd86fc1d5f9554ed55 is 50, key is test_row_0/A:col10/1731735655351/Put/seqid=0 2024-11-16T05:40:56,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742173_1349 (size=12301) 2024-11-16T05:40:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-16T05:40:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:56,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. as already flushing 2024-11-16T05:40:56,500 INFO [Thread-1187 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1187. 2024-11-16T05:40:56,500 DEBUG [Thread-1187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:56,500 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:56,500 INFO [Thread-1191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1191. 2024-11-16T05:40:56,500 DEBUG [Thread-1191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:56,500 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:56,500 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:56,500 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:56,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:56,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:56,501 INFO [Thread-1183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1183. 2024-11-16T05:40:56,501 DEBUG [Thread-1183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:56,501 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:56,501 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:56,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:56,501 INFO [Thread-1189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1189. 2024-11-16T05:40:56,501 DEBUG [Thread-1189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:56,501 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:56,501 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:56,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:56,501 INFO [Thread-1185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1185. 2024-11-16T05:40:56,501 DEBUG [Thread-1185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:40:56,501 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:56,502 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:56,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:56,580 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/414b4550257a4f0aa9cc52f1eb5b653f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/414b4550257a4f0aa9cc52f1eb5b653f 2024-11-16T05:40:56,581 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/0765ef70b107448c8719bf648607a7a0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/0765ef70b107448c8719bf648607a7a0 2024-11-16T05:40:56,584 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/B of e8755229bb6a5d7b53307633970aa7aa into 414b4550257a4f0aa9cc52f1eb5b653f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:56,584 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:56,584 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/B, priority=9, startTime=1731735656153; duration=0sec 2024-11-16T05:40:56,584 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:40:56,584 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:B 2024-11-16T05:40:56,585 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-16T05:40:56,586 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/A of e8755229bb6a5d7b53307633970aa7aa into 0765ef70b107448c8719bf648607a7a0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:56,586 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:56,586 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/A, priority=9, startTime=1731735656153; duration=0sec 2024-11-16T05:40:56,586 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:56,586 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:A 2024-11-16T05:40:56,586 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86857 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-16T05:40:56,587 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): e8755229bb6a5d7b53307633970aa7aa/C is initiating minor compaction (all files) 2024-11-16T05:40:56,587 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8755229bb6a5d7b53307633970aa7aa/C in TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:56,587 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/46661510f1034c4baa669764f8285b30, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e7012606f2d742c6b7adb0dc2be25cb1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/28e8ba265f38472ca3b5697f1020a52f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/f75f39a2dfa74c9bb29321ff08e282c2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3c3e50a2fb1a47cfb491cc2c3f126273, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b1009a0d90544d80bacd5a83ccb7aa58, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2d25dc792da04c0bb37c4864dbc8b128] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp, totalSize=84.8 K 2024-11-16T05:40:56,587 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46661510f1034c4baa669764f8285b30, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731735649595 2024-11-16T05:40:56,587 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7012606f2d742c6b7adb0dc2be25cb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731735649939 2024-11-16T05:40:56,587 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28e8ba265f38472ca3b5697f1020a52f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731735650592 2024-11-16T05:40:56,588 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f75f39a2dfa74c9bb29321ff08e282c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731735651756 2024-11-16T05:40:56,588 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c3e50a2fb1a47cfb491cc2c3f126273, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731735652952 2024-11-16T05:40:56,588 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1009a0d90544d80bacd5a83ccb7aa58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731735654128 2024-11-16T05:40:56,589 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d25dc792da04c0bb37c4864dbc8b128, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731735655270 2024-11-16T05:40:56,599 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8755229bb6a5d7b53307633970aa7aa#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:40:56,599 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/e01c7a91c8fe4dae8dfd549120a1ab3f is 50, key is test_row_0/C:col10/1731735655270/Put/seqid=0 2024-11-16T05:40:56,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742174_1350 (size=13289) 2024-11-16T05:40:56,639 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/eb294ff92d424efd86fc1d5f9554ed55 2024-11-16T05:40:56,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/2038fc90b9db42079a320c96c57b9bfc is 50, key is test_row_0/B:col10/1731735655351/Put/seqid=0 2024-11-16T05:40:56,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742175_1351 (size=12301) 2024-11-16T05:40:57,010 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/e01c7a91c8fe4dae8dfd549120a1ab3f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e01c7a91c8fe4dae8dfd549120a1ab3f 2024-11-16T05:40:57,016 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in e8755229bb6a5d7b53307633970aa7aa/C of e8755229bb6a5d7b53307633970aa7aa into e01c7a91c8fe4dae8dfd549120a1ab3f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:40:57,016 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:57,016 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., storeName=e8755229bb6a5d7b53307633970aa7aa/C, priority=9, startTime=1731735656154; duration=0sec 2024-11-16T05:40:57,016 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:40:57,016 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8755229bb6a5d7b53307633970aa7aa:C 2024-11-16T05:40:57,049 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/2038fc90b9db42079a320c96c57b9bfc 2024-11-16T05:40:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/baea32406c8e4eb48715c7275a8377d0 is 50, key is test_row_0/C:col10/1731735655351/Put/seqid=0 2024-11-16T05:40:57,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742176_1352 (size=12301) 2024-11-16T05:40:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-16T05:40:57,468 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/baea32406c8e4eb48715c7275a8377d0 2024-11-16T05:40:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/eb294ff92d424efd86fc1d5f9554ed55 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/eb294ff92d424efd86fc1d5f9554ed55 2024-11-16T05:40:57,478 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/eb294ff92d424efd86fc1d5f9554ed55, entries=150, sequenceid=459, filesize=12.0 K 2024-11-16T05:40:57,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/2038fc90b9db42079a320c96c57b9bfc as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/2038fc90b9db42079a320c96c57b9bfc 2024-11-16T05:40:57,485 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/2038fc90b9db42079a320c96c57b9bfc, entries=150, sequenceid=459, filesize=12.0 K 2024-11-16T05:40:57,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/baea32406c8e4eb48715c7275a8377d0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/baea32406c8e4eb48715c7275a8377d0 2024-11-16T05:40:57,492 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/baea32406c8e4eb48715c7275a8377d0, entries=150, sequenceid=459, filesize=12.0 K 2024-11-16T05:40:57,493 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for e8755229bb6a5d7b53307633970aa7aa in 1265ms, sequenceid=459, compaction requested=false 2024-11-16T05:40:57,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for e8755229bb6a5d7b53307633970aa7aa: 2024-11-16T05:40:57,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:57,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-16T05:40:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-16T05:40:57,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-16T05:40:57,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1870 sec 2024-11-16T05:40:57,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 2.1910 sec 2024-11-16T05:40:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-16T05:40:59,480 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:40:59,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:40:59,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3496 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3326 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3349 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3443 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3323 2024-11-16T05:40:59,481 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:40:59,481 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:40:59,482 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:40:59,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:59,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:40:59,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:40:59,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:40:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:40:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-16T05:40:59,491 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735659490"}]},"ts":"1731735659490"} 2024-11-16T05:40:59,493 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:40:59,493 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:40:59,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:40:59,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8755229bb6a5d7b53307633970aa7aa, UNASSIGN}] 2024-11-16T05:40:59,498 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8755229bb6a5d7b53307633970aa7aa, UNASSIGN 2024-11-16T05:40:59,499 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=e8755229bb6a5d7b53307633970aa7aa, regionState=CLOSING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:40:59,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8755229bb6a5d7b53307633970aa7aa, UNASSIGN because future has completed 2024-11-16T05:40:59,501 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:40:59,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:40:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-16T05:40:59,655 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:40:59,655 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:40:59,655 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing e8755229bb6a5d7b53307633970aa7aa, disabling compactions & flushes 2024-11-16T05:40:59,655 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:59,655 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:59,655 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. after waiting 0 ms 2024-11-16T05:40:59,655 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:40:59,655 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing e8755229bb6a5d7b53307633970aa7aa 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T05:40:59,661 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/93339490a5f14efa9db5ee69e9db0356 is 50, key is test_row_0/A:col10/1731735656500/Put/seqid=0 2024-11-16T05:40:59,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742177_1353 (size=9857) 2024-11-16T05:40:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-16T05:41:00,067 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/93339490a5f14efa9db5ee69e9db0356 2024-11-16T05:41:00,080 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/e1a6bfaf0bd946efbcdbefba9f3acb18 is 50, key is test_row_0/B:col10/1731735656500/Put/seqid=0 2024-11-16T05:41:00,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742178_1354 (size=9857) 2024-11-16T05:41:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-16T05:41:00,485 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/e1a6bfaf0bd946efbcdbefba9f3acb18 2024-11-16T05:41:00,497 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/8b85d0344c6043c29c672de0a90fe905 is 50, key is test_row_0/C:col10/1731735656500/Put/seqid=0 2024-11-16T05:41:00,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742179_1355 (size=9857) 2024-11-16T05:41:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-16T05:41:00,904 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/8b85d0344c6043c29c672de0a90fe905 2024-11-16T05:41:00,912 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/A/93339490a5f14efa9db5ee69e9db0356 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/93339490a5f14efa9db5ee69e9db0356 2024-11-16T05:41:00,916 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/93339490a5f14efa9db5ee69e9db0356, entries=100, sequenceid=470, filesize=9.6 K 2024-11-16T05:41:00,917 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/B/e1a6bfaf0bd946efbcdbefba9f3acb18 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e1a6bfaf0bd946efbcdbefba9f3acb18 2024-11-16T05:41:00,921 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e1a6bfaf0bd946efbcdbefba9f3acb18, entries=100, sequenceid=470, filesize=9.6 K 2024-11-16T05:41:00,922 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/.tmp/C/8b85d0344c6043c29c672de0a90fe905 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/8b85d0344c6043c29c672de0a90fe905 2024-11-16T05:41:00,925 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/8b85d0344c6043c29c672de0a90fe905, entries=100, sequenceid=470, filesize=9.6 K 2024-11-16T05:41:00,926 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e8755229bb6a5d7b53307633970aa7aa in 1271ms, sequenceid=470, compaction requested=true 2024-11-16T05:41:00,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/d2cf692e3ebd45c083f538b339aa393a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/58f660c70834476daaf5a7949676ea16, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cb232d62af6a466ca7ef14974e8b2774, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/322b9ee3a9fe4fae9bf79fd9d298532a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cc1f8be1997b49248b454e3ff4c6c420, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/ac380ccee4ce433c8cd436c1c3d20330, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/eeaf90d258404e879854818d0c7d4f00, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/9656687f86094f9e9f84558a6e76ba02, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b8b3f517dd534cb99b24ea896bf2f20e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/067889a4678d4d9888dcfd44c1558248, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/318aac580f8342f19d9bca74c5c28283, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/5c9804378e4c4980a1322223e88299b6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/0af5c97aae354cfda7d09a123cc58586, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b45b2f04728a41de86cb37b71cfe4efd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8e9b9af1195d44d28ef865b14d5bcf63, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8214ce84cfd740ba8a888562765ae2e2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/7f4f949732f94079abe46f52d572b3d6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/4f09e2b735be47be9a45e77cf0201195, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8dc451065e484fc58034dd1832e29cc6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c044b30cea884624a07d3fb3f9f0bf94, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c6d6d59fa0724e029b6f9c270e6adef2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9377c6ebae94290b7b6f18561e85497, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/6bc12925e476409195bcb4b89bd4d27d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f557752b6a804cea9251f8719801c0f1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c570fbf244224b0a9cee3e237feaf076, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/406e3f55a7d74eab89b1815eb69bac64, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/80186cfbc0a648728f8e1e5344f9c69c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/dd757944c9024cc791e24b74b51c8a27, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/72be5442763e49eb98032fc30f63b731, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9b5ca14374b4456ac16db875736804d] to archive 2024-11-16T05:41:00,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:41:00,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/d2cf692e3ebd45c083f538b339aa393a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/d2cf692e3ebd45c083f538b339aa393a 2024-11-16T05:41:00,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/58f660c70834476daaf5a7949676ea16 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/58f660c70834476daaf5a7949676ea16 2024-11-16T05:41:00,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cb232d62af6a466ca7ef14974e8b2774 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cb232d62af6a466ca7ef14974e8b2774 2024-11-16T05:41:00,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/322b9ee3a9fe4fae9bf79fd9d298532a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/322b9ee3a9fe4fae9bf79fd9d298532a 2024-11-16T05:41:00,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cc1f8be1997b49248b454e3ff4c6c420 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/cc1f8be1997b49248b454e3ff4c6c420 2024-11-16T05:41:00,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/ac380ccee4ce433c8cd436c1c3d20330 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/ac380ccee4ce433c8cd436c1c3d20330 2024-11-16T05:41:00,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/eeaf90d258404e879854818d0c7d4f00 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/eeaf90d258404e879854818d0c7d4f00 2024-11-16T05:41:00,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/9656687f86094f9e9f84558a6e76ba02 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/9656687f86094f9e9f84558a6e76ba02 2024-11-16T05:41:00,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b8b3f517dd534cb99b24ea896bf2f20e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b8b3f517dd534cb99b24ea896bf2f20e 2024-11-16T05:41:00,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/067889a4678d4d9888dcfd44c1558248 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/067889a4678d4d9888dcfd44c1558248 2024-11-16T05:41:00,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/318aac580f8342f19d9bca74c5c28283 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/318aac580f8342f19d9bca74c5c28283 2024-11-16T05:41:00,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/5c9804378e4c4980a1322223e88299b6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/5c9804378e4c4980a1322223e88299b6 2024-11-16T05:41:00,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/0af5c97aae354cfda7d09a123cc58586 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/0af5c97aae354cfda7d09a123cc58586 2024-11-16T05:41:00,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b45b2f04728a41de86cb37b71cfe4efd to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/b45b2f04728a41de86cb37b71cfe4efd 2024-11-16T05:41:00,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8e9b9af1195d44d28ef865b14d5bcf63 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8e9b9af1195d44d28ef865b14d5bcf63 2024-11-16T05:41:00,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8214ce84cfd740ba8a888562765ae2e2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8214ce84cfd740ba8a888562765ae2e2 2024-11-16T05:41:00,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/7f4f949732f94079abe46f52d572b3d6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/7f4f949732f94079abe46f52d572b3d6 2024-11-16T05:41:00,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/4f09e2b735be47be9a45e77cf0201195 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/4f09e2b735be47be9a45e77cf0201195 2024-11-16T05:41:00,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8dc451065e484fc58034dd1832e29cc6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/8dc451065e484fc58034dd1832e29cc6 2024-11-16T05:41:00,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c044b30cea884624a07d3fb3f9f0bf94 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c044b30cea884624a07d3fb3f9f0bf94 2024-11-16T05:41:00,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c6d6d59fa0724e029b6f9c270e6adef2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c6d6d59fa0724e029b6f9c270e6adef2 2024-11-16T05:41:00,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9377c6ebae94290b7b6f18561e85497 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9377c6ebae94290b7b6f18561e85497 2024-11-16T05:41:00,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/6bc12925e476409195bcb4b89bd4d27d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/6bc12925e476409195bcb4b89bd4d27d 2024-11-16T05:41:00,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f557752b6a804cea9251f8719801c0f1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f557752b6a804cea9251f8719801c0f1 2024-11-16T05:41:00,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c570fbf244224b0a9cee3e237feaf076 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/c570fbf244224b0a9cee3e237feaf076 2024-11-16T05:41:00,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/406e3f55a7d74eab89b1815eb69bac64 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/406e3f55a7d74eab89b1815eb69bac64 2024-11-16T05:41:00,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/80186cfbc0a648728f8e1e5344f9c69c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/80186cfbc0a648728f8e1e5344f9c69c 2024-11-16T05:41:00,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/dd757944c9024cc791e24b74b51c8a27 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/dd757944c9024cc791e24b74b51c8a27 2024-11-16T05:41:00,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/72be5442763e49eb98032fc30f63b731 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/72be5442763e49eb98032fc30f63b731 2024-11-16T05:41:00,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9b5ca14374b4456ac16db875736804d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/f9b5ca14374b4456ac16db875736804d 2024-11-16T05:41:00,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e6818e904a7a4449ac825c72498ea43a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ef541eb9c7844f5fa12029668741db3a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e4dd9242a10344da84144c1b87a81936, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/56afb1e949d44d6887e2d60616da8848, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/60b8167bd380489eaf0a9a09e0ec89a3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/d8e282fc06344990b0444d10892b189b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3c6026bf7b0046b998d6c980f3a7c5c3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/17c7aa3ba99546bd9bd9d73e5a710bd2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/131f7f88ef524e5c8c42a17d32a8bec6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1985b67e2a744596aa652c50dda68203, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/989dd8da1fd04291b56c1c4db134e83b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3e7a91a270f34fb6b29e5a9cfd7e0ed1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/2b0b3fdafb7648a9b8150b29e2d45f70, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/002ee3f8edf44be99438a604476b9bf9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1a44054bc88c41d08c182d04f18b4415, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/aa44576c9df54483abd548154341d0d9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/64f546b562354664bee325e399b4a8c2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a0dfede4c9ee4bd185617e35a3a86f15, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/cc7d0eccdc2e4841b38450d31c4e5475, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ec0af248500340fab0da3e191e85102f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/44701206dbce4402985cdb6d31a0554c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/849c072bef8a4db4a4addbbd54243bdc, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b90bc329db4e48dab288076e946ab0ec, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/09a9d6d08e6b4aeb97e73810bc90e27b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/181a722da3ce453ab4ee5bd3db650934, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/96c75296f06c433f9ddeb3499839d6cb, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a6b0119a7ffa46bca5b53ad99b266e1b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/573140ea3e7c4448a35608e3f202a3ab, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3eb99ca8a3ce4877bfe0fd91ef469ef0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b3ae2fac68d04cc39aa3d585e1c14c9f] to archive 2024-11-16T05:41:00,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:41:00,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e6818e904a7a4449ac825c72498ea43a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e6818e904a7a4449ac825c72498ea43a 2024-11-16T05:41:00,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ef541eb9c7844f5fa12029668741db3a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ef541eb9c7844f5fa12029668741db3a 2024-11-16T05:41:00,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e4dd9242a10344da84144c1b87a81936 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e4dd9242a10344da84144c1b87a81936 2024-11-16T05:41:00,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/56afb1e949d44d6887e2d60616da8848 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/56afb1e949d44d6887e2d60616da8848 2024-11-16T05:41:00,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/60b8167bd380489eaf0a9a09e0ec89a3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/60b8167bd380489eaf0a9a09e0ec89a3 2024-11-16T05:41:00,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/d8e282fc06344990b0444d10892b189b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/d8e282fc06344990b0444d10892b189b 2024-11-16T05:41:00,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3c6026bf7b0046b998d6c980f3a7c5c3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3c6026bf7b0046b998d6c980f3a7c5c3 2024-11-16T05:41:00,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/17c7aa3ba99546bd9bd9d73e5a710bd2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/17c7aa3ba99546bd9bd9d73e5a710bd2 2024-11-16T05:41:00,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/131f7f88ef524e5c8c42a17d32a8bec6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/131f7f88ef524e5c8c42a17d32a8bec6 2024-11-16T05:41:00,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1985b67e2a744596aa652c50dda68203 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1985b67e2a744596aa652c50dda68203 2024-11-16T05:41:00,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/989dd8da1fd04291b56c1c4db134e83b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/989dd8da1fd04291b56c1c4db134e83b 2024-11-16T05:41:00,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3e7a91a270f34fb6b29e5a9cfd7e0ed1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3e7a91a270f34fb6b29e5a9cfd7e0ed1 2024-11-16T05:41:00,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/2b0b3fdafb7648a9b8150b29e2d45f70 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/2b0b3fdafb7648a9b8150b29e2d45f70 2024-11-16T05:41:00,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/002ee3f8edf44be99438a604476b9bf9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/002ee3f8edf44be99438a604476b9bf9 2024-11-16T05:41:00,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1a44054bc88c41d08c182d04f18b4415 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/1a44054bc88c41d08c182d04f18b4415 2024-11-16T05:41:00,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/aa44576c9df54483abd548154341d0d9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/aa44576c9df54483abd548154341d0d9 2024-11-16T05:41:00,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/64f546b562354664bee325e399b4a8c2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/64f546b562354664bee325e399b4a8c2 2024-11-16T05:41:00,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a0dfede4c9ee4bd185617e35a3a86f15 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a0dfede4c9ee4bd185617e35a3a86f15 2024-11-16T05:41:00,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/cc7d0eccdc2e4841b38450d31c4e5475 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/cc7d0eccdc2e4841b38450d31c4e5475 2024-11-16T05:41:00,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ec0af248500340fab0da3e191e85102f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/ec0af248500340fab0da3e191e85102f 2024-11-16T05:41:00,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/44701206dbce4402985cdb6d31a0554c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/44701206dbce4402985cdb6d31a0554c 2024-11-16T05:41:00,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/849c072bef8a4db4a4addbbd54243bdc to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/849c072bef8a4db4a4addbbd54243bdc 2024-11-16T05:41:00,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b90bc329db4e48dab288076e946ab0ec to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b90bc329db4e48dab288076e946ab0ec 2024-11-16T05:41:00,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/09a9d6d08e6b4aeb97e73810bc90e27b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/09a9d6d08e6b4aeb97e73810bc90e27b 2024-11-16T05:41:00,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/181a722da3ce453ab4ee5bd3db650934 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/181a722da3ce453ab4ee5bd3db650934 2024-11-16T05:41:00,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/96c75296f06c433f9ddeb3499839d6cb to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/96c75296f06c433f9ddeb3499839d6cb 2024-11-16T05:41:00,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a6b0119a7ffa46bca5b53ad99b266e1b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/a6b0119a7ffa46bca5b53ad99b266e1b 2024-11-16T05:41:00,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/573140ea3e7c4448a35608e3f202a3ab to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/573140ea3e7c4448a35608e3f202a3ab 2024-11-16T05:41:00,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3eb99ca8a3ce4877bfe0fd91ef469ef0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/3eb99ca8a3ce4877bfe0fd91ef469ef0 2024-11-16T05:41:00,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b3ae2fac68d04cc39aa3d585e1c14c9f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/b3ae2fac68d04cc39aa3d585e1c14c9f 2024-11-16T05:41:00,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/5922202fe711438385f33df1b530a8a6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d0d3f2df24cb44099b5ba8bf075c02b0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1b3bc292891b40b4b8ed4a8d22495ff0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b9044d18ce414fa89ef2b96babc57898, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/edcf44b1a19c4cd088daa38b5f1db23f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ead3b1833daf4d10810a7b533c828000, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a6295ec500194dd3b31101c0abbf0bb6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ff56357e0a24426c85bc6d659519abfd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2c234a1d3d584fd19bf0f2aca998f24d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a45b6a3777e54e0c8d1023f89f1467ea, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3ade528461374f70b024cad06ef4d2da, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3b80a4425793443e94111f28c92a9403, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/cfb9ecc59cf84718a97058e9c541b7a8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/00bdfa16ddb24d26a85a4a6aa28b86e8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/393bd21acad9468ca46dd1d754eb58a3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d1d89f0b5bb74806b8985e74e8c46064, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ba1c5e99911345478c56269df53c1b2c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e1334497d44c451aab317b656ab09ce3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/47fd85d11ae64dbaabc3667703fff5ee, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/385b96833e4c4aa7b8871a6cbb29bf62, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/4777508fece94de691f510a69d2f5124, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/46661510f1034c4baa669764f8285b30, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1130dfb219054bd09f3417062b16771c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e7012606f2d742c6b7adb0dc2be25cb1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/28e8ba265f38472ca3b5697f1020a52f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/f75f39a2dfa74c9bb29321ff08e282c2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3c3e50a2fb1a47cfb491cc2c3f126273, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b1009a0d90544d80bacd5a83ccb7aa58, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2d25dc792da04c0bb37c4864dbc8b128] to archive 2024-11-16T05:41:00,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:41:00,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/5922202fe711438385f33df1b530a8a6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/5922202fe711438385f33df1b530a8a6 2024-11-16T05:41:00,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d0d3f2df24cb44099b5ba8bf075c02b0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d0d3f2df24cb44099b5ba8bf075c02b0 2024-11-16T05:41:00,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1b3bc292891b40b4b8ed4a8d22495ff0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1b3bc292891b40b4b8ed4a8d22495ff0 2024-11-16T05:41:00,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b9044d18ce414fa89ef2b96babc57898 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b9044d18ce414fa89ef2b96babc57898 2024-11-16T05:41:00,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/edcf44b1a19c4cd088daa38b5f1db23f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/edcf44b1a19c4cd088daa38b5f1db23f 2024-11-16T05:41:00,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ead3b1833daf4d10810a7b533c828000 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ead3b1833daf4d10810a7b533c828000 2024-11-16T05:41:00,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a6295ec500194dd3b31101c0abbf0bb6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a6295ec500194dd3b31101c0abbf0bb6 2024-11-16T05:41:00,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ff56357e0a24426c85bc6d659519abfd to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ff56357e0a24426c85bc6d659519abfd 2024-11-16T05:41:00,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2c234a1d3d584fd19bf0f2aca998f24d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2c234a1d3d584fd19bf0f2aca998f24d 2024-11-16T05:41:00,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a45b6a3777e54e0c8d1023f89f1467ea to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/a45b6a3777e54e0c8d1023f89f1467ea 2024-11-16T05:41:01,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3ade528461374f70b024cad06ef4d2da to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3ade528461374f70b024cad06ef4d2da 2024-11-16T05:41:01,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3b80a4425793443e94111f28c92a9403 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3b80a4425793443e94111f28c92a9403 2024-11-16T05:41:01,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/cfb9ecc59cf84718a97058e9c541b7a8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/cfb9ecc59cf84718a97058e9c541b7a8 2024-11-16T05:41:01,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/00bdfa16ddb24d26a85a4a6aa28b86e8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/00bdfa16ddb24d26a85a4a6aa28b86e8 2024-11-16T05:41:01,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/393bd21acad9468ca46dd1d754eb58a3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/393bd21acad9468ca46dd1d754eb58a3 2024-11-16T05:41:01,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d1d89f0b5bb74806b8985e74e8c46064 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/d1d89f0b5bb74806b8985e74e8c46064 2024-11-16T05:41:01,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ba1c5e99911345478c56269df53c1b2c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/ba1c5e99911345478c56269df53c1b2c 2024-11-16T05:41:01,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e1334497d44c451aab317b656ab09ce3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e1334497d44c451aab317b656ab09ce3 2024-11-16T05:41:01,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/47fd85d11ae64dbaabc3667703fff5ee to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/47fd85d11ae64dbaabc3667703fff5ee 2024-11-16T05:41:01,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/385b96833e4c4aa7b8871a6cbb29bf62 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/385b96833e4c4aa7b8871a6cbb29bf62 2024-11-16T05:41:01,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/4777508fece94de691f510a69d2f5124 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/4777508fece94de691f510a69d2f5124 2024-11-16T05:41:01,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/46661510f1034c4baa669764f8285b30 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/46661510f1034c4baa669764f8285b30 2024-11-16T05:41:01,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1130dfb219054bd09f3417062b16771c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/1130dfb219054bd09f3417062b16771c 2024-11-16T05:41:01,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e7012606f2d742c6b7adb0dc2be25cb1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e7012606f2d742c6b7adb0dc2be25cb1 2024-11-16T05:41:01,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/28e8ba265f38472ca3b5697f1020a52f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/28e8ba265f38472ca3b5697f1020a52f 2024-11-16T05:41:01,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/f75f39a2dfa74c9bb29321ff08e282c2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/f75f39a2dfa74c9bb29321ff08e282c2 2024-11-16T05:41:01,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3c3e50a2fb1a47cfb491cc2c3f126273 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/3c3e50a2fb1a47cfb491cc2c3f126273 2024-11-16T05:41:01,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b1009a0d90544d80bacd5a83ccb7aa58 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/b1009a0d90544d80bacd5a83ccb7aa58 2024-11-16T05:41:01,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2d25dc792da04c0bb37c4864dbc8b128 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/2d25dc792da04c0bb37c4864dbc8b128 2024-11-16T05:41:01,022 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/recovered.edits/473.seqid, newMaxSeqId=473, maxSeqId=1 2024-11-16T05:41:01,023 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. 2024-11-16T05:41:01,023 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for e8755229bb6a5d7b53307633970aa7aa: Waiting for close lock at 1731735659655Running coprocessor pre-close hooks at 1731735659655Disabling compacts and flushes for region at 1731735659655Disabling writes for close at 1731735659655Obtaining lock to block concurrent updates at 1731735659655Preparing flush snapshotting stores in e8755229bb6a5d7b53307633970aa7aa at 1731735659655Finished memstore snapshotting TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731735659656 (+1 ms)Flushing stores of TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa. at 1731735659656Flushing e8755229bb6a5d7b53307633970aa7aa/A: creating writer at 1731735659657 (+1 ms)Flushing e8755229bb6a5d7b53307633970aa7aa/A: appending metadata at 1731735659660 (+3 ms)Flushing e8755229bb6a5d7b53307633970aa7aa/A: closing flushed file at 1731735659660Flushing e8755229bb6a5d7b53307633970aa7aa/B: creating writer at 1731735660077 (+417 ms)Flushing e8755229bb6a5d7b53307633970aa7aa/B: appending metadata at 1731735660080 (+3 ms)Flushing e8755229bb6a5d7b53307633970aa7aa/B: closing flushed file at 1731735660080Flushing e8755229bb6a5d7b53307633970aa7aa/C: creating writer at 1731735660491 (+411 ms)Flushing e8755229bb6a5d7b53307633970aa7aa/C: appending metadata at 1731735660496 (+5 ms)Flushing e8755229bb6a5d7b53307633970aa7aa/C: closing flushed file at 1731735660496Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77a91df5: reopening flushed file at 1731735660911 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35d0aee1: reopening flushed file at 1731735660916 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10307a1: reopening flushed file at 1731735660921 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e8755229bb6a5d7b53307633970aa7aa in 1271ms, sequenceid=470, compaction requested=true at 1731735660926 (+5 ms)Writing region close event to WAL at 1731735661020 (+94 ms)Running coprocessor post-close hooks at 1731735661023 (+3 ms)Closed at 1731735661023 2024-11-16T05:41:01,025 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:41:01,025 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=e8755229bb6a5d7b53307633970aa7aa, regionState=CLOSED 2024-11-16T05:41:01,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:41:01,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-16T05:41:01,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure e8755229bb6a5d7b53307633970aa7aa, server=3456ee6a3164,35555,1731735575219 in 1.5260 sec 2024-11-16T05:41:01,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-16T05:41:01,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8755229bb6a5d7b53307633970aa7aa, UNASSIGN in 1.5330 sec 2024-11-16T05:41:01,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-16T05:41:01,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5370 sec 2024-11-16T05:41:01,034 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735661034"}]},"ts":"1731735661034"} 2024-11-16T05:41:01,036 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:41:01,036 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:41:01,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5520 sec 2024-11-16T05:41:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-16T05:41:01,639 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:01,640 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:41:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:01,644 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-16T05:41:01,646 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:01,651 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:41:01,655 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/recovered.edits] 2024-11-16T05:41:01,658 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/0765ef70b107448c8719bf648607a7a0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/0765ef70b107448c8719bf648607a7a0 2024-11-16T05:41:01,660 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/93339490a5f14efa9db5ee69e9db0356 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/93339490a5f14efa9db5ee69e9db0356 2024-11-16T05:41:01,661 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/eb294ff92d424efd86fc1d5f9554ed55 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/A/eb294ff92d424efd86fc1d5f9554ed55 2024-11-16T05:41:01,664 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/2038fc90b9db42079a320c96c57b9bfc to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/2038fc90b9db42079a320c96c57b9bfc 2024-11-16T05:41:01,665 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/414b4550257a4f0aa9cc52f1eb5b653f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/414b4550257a4f0aa9cc52f1eb5b653f 2024-11-16T05:41:01,666 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e1a6bfaf0bd946efbcdbefba9f3acb18 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/B/e1a6bfaf0bd946efbcdbefba9f3acb18 2024-11-16T05:41:01,668 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/8b85d0344c6043c29c672de0a90fe905 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/8b85d0344c6043c29c672de0a90fe905 2024-11-16T05:41:01,668 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/baea32406c8e4eb48715c7275a8377d0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/baea32406c8e4eb48715c7275a8377d0 2024-11-16T05:41:01,669 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e01c7a91c8fe4dae8dfd549120a1ab3f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/C/e01c7a91c8fe4dae8dfd549120a1ab3f 2024-11-16T05:41:01,672 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/recovered.edits/473.seqid to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa/recovered.edits/473.seqid 2024-11-16T05:41:01,672 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/e8755229bb6a5d7b53307633970aa7aa 2024-11-16T05:41:01,672 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:41:01,674 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:01,677 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:41:01,679 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:41:01,680 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:01,680 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:41:01,680 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731735661680"}]},"ts":"9223372036854775807"} 2024-11-16T05:41:01,682 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:41:01,682 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e8755229bb6a5d7b53307633970aa7aa, NAME => 'TestAcidGuarantees,,1731735634171.e8755229bb6a5d7b53307633970aa7aa.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:41:01,682 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:41:01,682 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731735661682"}]},"ts":"9223372036854775807"} 2024-11-16T05:41:01,684 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:41:01,684 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:01,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-16T05:41:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-16T05:41:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:41:01,749 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:01,761 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241 (was 241), OpenFileDescriptor=455 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=481 (was 578), ProcessCount=11 (was 11), AvailableMemoryMB=3610 (was 3680) 2024-11-16T05:41:01,770 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=481, ProcessCount=11, AvailableMemoryMB=3609 2024-11-16T05:41:01,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:41:01,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:41:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:01,774 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:41:01,774 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:01,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-16T05:41:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:41:01,775 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:41:01,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742180_1356 (size=959) 2024-11-16T05:41:01,784 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d 2024-11-16T05:41:01,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742181_1357 (size=53) 2024-11-16T05:41:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:41:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:41:02,191 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:41:02,192 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4d436cb8d7fad8c1b05b38c8375a7637, disabling compactions & flushes 2024-11-16T05:41:02,192 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:02,192 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:02,192 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. after waiting 0 ms 2024-11-16T05:41:02,192 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:02,192 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:02,192 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4d436cb8d7fad8c1b05b38c8375a7637: Waiting for close lock at 1731735662192Disabling compacts and flushes for region at 1731735662192Disabling writes for close at 1731735662192Writing region close event to WAL at 1731735662192Closed at 1731735662192 2024-11-16T05:41:02,194 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:41:02,194 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731735662194"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731735662194"}]},"ts":"1731735662194"} 2024-11-16T05:41:02,197 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:41:02,199 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:41:02,200 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735662200"}]},"ts":"1731735662200"} 2024-11-16T05:41:02,203 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:41:02,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, ASSIGN}] 2024-11-16T05:41:02,205 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, ASSIGN 2024-11-16T05:41:02,206 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, ASSIGN; state=OFFLINE, location=3456ee6a3164,35555,1731735575219; forceNewPlan=false, retain=false 2024-11-16T05:41:02,358 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=4d436cb8d7fad8c1b05b38c8375a7637, regionState=OPENING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:02,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, ASSIGN because future has completed 2024-11-16T05:41:02,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:41:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:41:02,520 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:02,520 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:41:02,521 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,521 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:41:02,521 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,521 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,523 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,526 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d436cb8d7fad8c1b05b38c8375a7637 columnFamilyName A 2024-11-16T05:41:02,526 DEBUG [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:02,527 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(327): Store=4d436cb8d7fad8c1b05b38c8375a7637/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:41:02,527 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,529 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d436cb8d7fad8c1b05b38c8375a7637 columnFamilyName B 2024-11-16T05:41:02,529 DEBUG [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:02,529 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(327): Store=4d436cb8d7fad8c1b05b38c8375a7637/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:41:02,529 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,530 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d436cb8d7fad8c1b05b38c8375a7637 columnFamilyName C 2024-11-16T05:41:02,530 DEBUG [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:02,531 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(327): Store=4d436cb8d7fad8c1b05b38c8375a7637/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:41:02,531 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,531 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,531 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,532 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,532 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,533 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:41:02,534 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,535 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:41:02,536 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened 4d436cb8d7fad8c1b05b38c8375a7637; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73024332, jitterRate=0.08814734220504761}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:41:02,536 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:02,536 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for 4d436cb8d7fad8c1b05b38c8375a7637: Running coprocessor pre-open hook at 1731735662521Writing region info on filesystem at 1731735662521Initializing all the Stores at 1731735662523 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735662523Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735662523Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735662523Cleaning up temporary data from old regions at 1731735662532 (+9 ms)Running coprocessor post-open hooks at 1731735662536 (+4 ms)Region opened successfully at 1731735662536 2024-11-16T05:41:02,537 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., pid=99, masterSystemTime=1731735662515 2024-11-16T05:41:02,538 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:02,539 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:02,539 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=4d436cb8d7fad8c1b05b38c8375a7637, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:02,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:41:02,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-16T05:41:02,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 in 179 msec 2024-11-16T05:41:02,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-16T05:41:02,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, ASSIGN in 339 msec 2024-11-16T05:41:02,545 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:41:02,545 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735662545"}]},"ts":"1731735662545"} 2024-11-16T05:41:02,546 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:41:02,547 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:41:02,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 775 msec 2024-11-16T05:41:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:41:02,909 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:02,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:02,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:02,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:02,911 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ad2879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:02,913 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a83cb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:02,914 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:02,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:02,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:02,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:02,917 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:41:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,39409,1731735574536 2024-11-16T05:41:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@398aed24 2024-11-16T05:41:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:41:02,919 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:41:02,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:41:02,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:41:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:02,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742182_1358 (size=995) 2024-11-16T05:41:03,331 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T05:41:03,331 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T05:41:03,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:41:03,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, REOPEN/MOVE}] 2024-11-16T05:41:03,338 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, REOPEN/MOVE 2024-11-16T05:41:03,338 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=4d436cb8d7fad8c1b05b38c8375a7637, regionState=CLOSING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:03,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, REOPEN/MOVE because future has completed 2024-11-16T05:41:03,340 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:41:03,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:41:03,493 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,494 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:41:03,494 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing 4d436cb8d7fad8c1b05b38c8375a7637, disabling compactions & flushes 2024-11-16T05:41:03,494 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:03,494 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:03,494 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. after waiting 0 ms 2024-11-16T05:41:03,494 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:03,500 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:41:03,501 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:03,501 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for 4d436cb8d7fad8c1b05b38c8375a7637: Waiting for close lock at 1731735663494Running coprocessor pre-close hooks at 1731735663494Disabling compacts and flushes for region at 1731735663494Disabling writes for close at 1731735663494Writing region close event to WAL at 1731735663494Running coprocessor post-close hooks at 1731735663501 (+7 ms)Closed at 1731735663501 2024-11-16T05:41:03,501 WARN [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: 4d436cb8d7fad8c1b05b38c8375a7637 to self. 2024-11-16T05:41:03,504 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,505 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=4d436cb8d7fad8c1b05b38c8375a7637, regionState=CLOSED 2024-11-16T05:41:03,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:41:03,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-16T05:41:03,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 in 168 msec 2024-11-16T05:41:03,510 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, REOPEN/MOVE; state=CLOSED, location=3456ee6a3164,35555,1731735575219; forceNewPlan=false, retain=true 2024-11-16T05:41:03,555 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:41:03,661 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=4d436cb8d7fad8c1b05b38c8375a7637, regionState=OPENING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:03,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, REOPEN/MOVE because future has completed 2024-11-16T05:41:03,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:41:03,820 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:03,821 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:41:03,822 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,822 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:41:03,822 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,822 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,826 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,828 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d436cb8d7fad8c1b05b38c8375a7637 columnFamilyName A 2024-11-16T05:41:03,829 DEBUG [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:03,830 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(327): Store=4d436cb8d7fad8c1b05b38c8375a7637/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:41:03,830 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,831 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d436cb8d7fad8c1b05b38c8375a7637 columnFamilyName B 2024-11-16T05:41:03,831 DEBUG [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:03,831 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(327): Store=4d436cb8d7fad8c1b05b38c8375a7637/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:41:03,831 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,832 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d436cb8d7fad8c1b05b38c8375a7637 columnFamilyName C 2024-11-16T05:41:03,832 DEBUG [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:03,832 INFO [StoreOpener-4d436cb8d7fad8c1b05b38c8375a7637-1 {}] regionserver.HStore(327): Store=4d436cb8d7fad8c1b05b38c8375a7637/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:41:03,832 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,833 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,833 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,834 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,834 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,835 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:41:03,835 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,836 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened 4d436cb8d7fad8c1b05b38c8375a7637; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62172818, jitterRate=-0.07355281710624695}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:41:03,836 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,837 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for 4d436cb8d7fad8c1b05b38c8375a7637: Running coprocessor pre-open hook at 1731735663822Writing region info on filesystem at 1731735663822Initializing all the Stores at 1731735663824 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735663825 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735663825Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735663825Cleaning up temporary data from old regions at 1731735663834 (+9 ms)Running coprocessor post-open hooks at 1731735663836 (+2 ms)Region opened successfully at 1731735663836 2024-11-16T05:41:03,837 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., pid=104, masterSystemTime=1731735663814 2024-11-16T05:41:03,839 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:03,839 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:03,839 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=4d436cb8d7fad8c1b05b38c8375a7637, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:03,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:41:03,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-16T05:41:03,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 in 179 msec 2024-11-16T05:41:03,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-16T05:41:03,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, REOPEN/MOVE in 506 msec 2024-11-16T05:41:03,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-16T05:41:03,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-11-16T05:41:03,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 925 msec 2024-11-16T05:41:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-16T05:41:03,848 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:03,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:03,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:03,850 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@144ecf07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:03,851 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23bb5317, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:03,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:03,853 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224150dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:03,854 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344c6c20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:03,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:03,856 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c62a58d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:03,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@afd9b9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:03,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:03,859 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c1c75a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:03,861 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d18249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:03,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:03,863 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@245810c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:03,864 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd64afa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:03,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:03,866 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b1f696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:03,867 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3925d562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:03,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:03,869 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68de30e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:03,871 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6182a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:03,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:03,873 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e4b4f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:03,875 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454d6c8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:03,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:03,877 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6857f8e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:03,878 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695c83d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:03,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:03,881 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1991de99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:03,883 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c237359, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:03,885 DEBUG [Thread-1618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:03,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:03,885 DEBUG [Thread-1631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:03,885 DEBUG [Thread-1629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:03,885 DEBUG [Thread-1627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:03,886 DEBUG [Thread-1622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:03,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:03,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:03,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-16T05:41:03,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:41:03,889 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:03,890 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:03,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:03,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:41:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:41:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:41:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:41:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:41:03,897 DEBUG [Thread-1625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:03,897 DEBUG [Thread-1616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:03,897 DEBUG [Thread-1620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:03,897 DEBUG [Thread-1614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:03,899 DEBUG [Thread-1620 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:03,899 DEBUG [Thread-1620 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:03,899 DEBUG [Thread-1616 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:03,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:03,900 DEBUG [Thread-1616 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:03,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:03,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:03,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:41:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:41:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:41:03,905 DEBUG [Thread-1633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:41:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,906 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:03,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:41:03,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:41:03,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116da478181c5064afa93071564e2c7bbbf_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735663901/Put/seqid=0 2024-11-16T05:41:03,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742183_1359 (size=17034) 2024-11-16T05:41:03,940 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:03,944 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116da478181c5064afa93071564e2c7bbbf_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da478181c5064afa93071564e2c7bbbf_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:03,945 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/8ac3812fc0b94290be5f93838a978b91, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:03,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/8ac3812fc0b94290be5f93838a978b91 is 175, key is test_row_0/A:col10/1731735663901/Put/seqid=0 2024-11-16T05:41:03,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742184_1360 (size=48139) 2024-11-16T05:41:03,949 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/8ac3812fc0b94290be5f93838a978b91 2024-11-16T05:41:03,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:03,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735723947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:03,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735723951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:03,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735723951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:03,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735723952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:03,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735723954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:03,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/269daa25fd2e4526aae338deb366125a is 50, key is test_row_0/B:col10/1731735663901/Put/seqid=0 2024-11-16T05:41:03,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742185_1361 (size=12001) 2024-11-16T05:41:03,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/269daa25fd2e4526aae338deb366125a 2024-11-16T05:41:03,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/4a27e63849be44bbbb03f2783f77c51d is 50, key is test_row_0/C:col10/1731735663901/Put/seqid=0 2024-11-16T05:41:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:41:04,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742186_1362 (size=12001) 2024-11-16T05:41:04,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T05:41:04,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:04,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:04,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:04,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:04,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735724058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735724068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735724068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735724068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735724068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T05:41:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:04,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:41:04,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735724268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735724278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735724278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735724278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735724278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T05:41:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:04,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:04,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:04,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/4a27e63849be44bbbb03f2783f77c51d 2024-11-16T05:41:04,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/8ac3812fc0b94290be5f93838a978b91 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/8ac3812fc0b94290be5f93838a978b91 2024-11-16T05:41:04,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/8ac3812fc0b94290be5f93838a978b91, entries=250, sequenceid=16, filesize=47.0 K 2024-11-16T05:41:04,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/269daa25fd2e4526aae338deb366125a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/269daa25fd2e4526aae338deb366125a 2024-11-16T05:41:04,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/269daa25fd2e4526aae338deb366125a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T05:41:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/4a27e63849be44bbbb03f2783f77c51d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/4a27e63849be44bbbb03f2783f77c51d 2024-11-16T05:41:04,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/4a27e63849be44bbbb03f2783f77c51d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T05:41:04,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4d436cb8d7fad8c1b05b38c8375a7637 in 519ms, sequenceid=16, compaction requested=false 2024-11-16T05:41:04,426 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T05:41:04,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:04,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T05:41:04,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:04,503 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T05:41:04,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ec8fa1d899a445afb3fc53d18e59d22e_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735663949/Put/seqid=0 2024-11-16T05:41:04,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742187_1363 (size=12154) 2024-11-16T05:41:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:41:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:04,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735724595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735724596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735724597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735724598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735724599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735724708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735724718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735724718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735724718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735724718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:04,918 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ec8fa1d899a445afb3fc53d18e59d22e_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ec8fa1d899a445afb3fc53d18e59d22e_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:04,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/721bdd7c73084b4884cdb7e49d05652f, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:04,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/721bdd7c73084b4884cdb7e49d05652f is 175, key is test_row_0/A:col10/1731735663949/Put/seqid=0 2024-11-16T05:41:04,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742188_1364 (size=30955) 2024-11-16T05:41:04,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735724918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735724938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735724938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735724938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735724938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:41:05,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735725228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:05,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735725248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735725248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735725248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735725248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:05,323 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/721bdd7c73084b4884cdb7e49d05652f 2024-11-16T05:41:05,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/94fca42f257a412b9f7ebed3d0dbfb2b is 50, key is test_row_0/B:col10/1731735663949/Put/seqid=0 2024-11-16T05:41:05,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742189_1365 (size=12001) 2024-11-16T05:41:05,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735725738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:05,743 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/94fca42f257a412b9f7ebed3d0dbfb2b 2024-11-16T05:41:05,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/c29c93541c4a47bbb71f4eda94b42225 is 50, key is test_row_0/C:col10/1731735663949/Put/seqid=0 2024-11-16T05:41:05,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742190_1366 (size=12001) 2024-11-16T05:41:05,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735725758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735725758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735725758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:05,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735725758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:41:06,154 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/c29c93541c4a47bbb71f4eda94b42225 2024-11-16T05:41:06,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/721bdd7c73084b4884cdb7e49d05652f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/721bdd7c73084b4884cdb7e49d05652f 2024-11-16T05:41:06,160 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/721bdd7c73084b4884cdb7e49d05652f, entries=150, sequenceid=40, filesize=30.2 K 2024-11-16T05:41:06,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/94fca42f257a412b9f7ebed3d0dbfb2b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/94fca42f257a412b9f7ebed3d0dbfb2b 2024-11-16T05:41:06,166 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/94fca42f257a412b9f7ebed3d0dbfb2b, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T05:41:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/c29c93541c4a47bbb71f4eda94b42225 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c29c93541c4a47bbb71f4eda94b42225 2024-11-16T05:41:06,173 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c29c93541c4a47bbb71f4eda94b42225, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T05:41:06,173 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1670ms, sequenceid=40, compaction requested=false 2024-11-16T05:41:06,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:06,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:06,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-16T05:41:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-16T05:41:06,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-16T05:41:06,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2850 sec 2024-11-16T05:41:06,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.2920 sec 2024-11-16T05:41:06,552 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:41:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:06,753 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:06,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T05:41:06,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111617e964fd748c4fd1aa7002512575ac67_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735664595/Put/seqid=0 2024-11-16T05:41:06,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742191_1367 (size=14594) 2024-11-16T05:41:06,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735726830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:06,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735726831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:06,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735726832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735726846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735726847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:06,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:06,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:06,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:06,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735726968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735726968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,172 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:07,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735727168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735727168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735727168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,176 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111617e964fd748c4fd1aa7002512575ac67_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111617e964fd748c4fd1aa7002512575ac67_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:07,177 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/0e7de23e5bc8405cb3d168528760940c, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:07,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/0e7de23e5bc8405cb3d168528760940c is 175, key is test_row_0/A:col10/1731735664595/Put/seqid=0 2024-11-16T05:41:07,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742192_1368 (size=39549) 2024-11-16T05:41:07,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735727178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735727178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735727478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735727478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735727478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735727488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735727488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,581 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/0e7de23e5bc8405cb3d168528760940c 2024-11-16T05:41:07,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/d63656706b1b41c0a61bea7438aa26a7 is 50, key is test_row_0/B:col10/1731735664595/Put/seqid=0 2024-11-16T05:41:07,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742193_1369 (size=12001) 2024-11-16T05:41:07,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:07,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735727988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:07,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/d63656706b1b41c0a61bea7438aa26a7 2024-11-16T05:41:08,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/c7b674c0c62e4d01b32627a66459d2fd is 50, key is test_row_0/C:col10/1731735664595/Put/seqid=0 2024-11-16T05:41:08,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735727998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:08,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735727998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:08,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735727998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:08,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742194_1370 (size=12001) 2024-11-16T05:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:08,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735728008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:08,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:41:08,048 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:08,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:08,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-16T05:41:08,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T05:41:08,050 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:08,051 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:08,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T05:41:08,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T05:41:08,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:08,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:08,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:08,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:08,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:08,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T05:41:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:08,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T05:41:08,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/c7b674c0c62e4d01b32627a66459d2fd 2024-11-16T05:41:08,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/0e7de23e5bc8405cb3d168528760940c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0e7de23e5bc8405cb3d168528760940c 2024-11-16T05:41:08,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0e7de23e5bc8405cb3d168528760940c, entries=200, sequenceid=53, filesize=38.6 K 2024-11-16T05:41:08,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/d63656706b1b41c0a61bea7438aa26a7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d63656706b1b41c0a61bea7438aa26a7 2024-11-16T05:41:08,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d63656706b1b41c0a61bea7438aa26a7, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T05:41:08,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/c7b674c0c62e4d01b32627a66459d2fd as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c7b674c0c62e4d01b32627a66459d2fd 2024-11-16T05:41:08,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c7b674c0c62e4d01b32627a66459d2fd, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T05:41:08,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1666ms, sequenceid=53, compaction requested=true 2024-11-16T05:41:08,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:08,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:08,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:08,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:08,419 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:08,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:08,419 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:08,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:08,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:08,420 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:08,420 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:08,420 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/B is initiating minor compaction (all files) 2024-11-16T05:41:08,420 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/A is initiating minor compaction (all files) 2024-11-16T05:41:08,420 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/B in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:08,420 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/A in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:08,420 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/8ac3812fc0b94290be5f93838a978b91, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/721bdd7c73084b4884cdb7e49d05652f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0e7de23e5bc8405cb3d168528760940c] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=115.9 K 2024-11-16T05:41:08,420 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/269daa25fd2e4526aae338deb366125a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/94fca42f257a412b9f7ebed3d0dbfb2b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d63656706b1b41c0a61bea7438aa26a7] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=35.2 K 2024-11-16T05:41:08,420 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:08,420 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/8ac3812fc0b94290be5f93838a978b91, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/721bdd7c73084b4884cdb7e49d05652f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0e7de23e5bc8405cb3d168528760940c] 2024-11-16T05:41:08,421 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 269daa25fd2e4526aae338deb366125a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731735663901 2024-11-16T05:41:08,421 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ac3812fc0b94290be5f93838a978b91, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731735663901 2024-11-16T05:41:08,421 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 94fca42f257a412b9f7ebed3d0dbfb2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731735663949 2024-11-16T05:41:08,421 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721bdd7c73084b4884cdb7e49d05652f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731735663949 2024-11-16T05:41:08,422 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e7de23e5bc8405cb3d168528760940c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735664595 2024-11-16T05:41:08,422 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting d63656706b1b41c0a61bea7438aa26a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735664595 2024-11-16T05:41:08,428 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:08,429 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:08,429 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/3096e165422647d5a14424badf70bf8f is 50, key is test_row_0/B:col10/1731735664595/Put/seqid=0 2024-11-16T05:41:08,431 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116341109001de14d99b56d355d6b7abb9b_4d436cb8d7fad8c1b05b38c8375a7637 store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:08,433 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116341109001de14d99b56d355d6b7abb9b_4d436cb8d7fad8c1b05b38c8375a7637, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:08,434 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116341109001de14d99b56d355d6b7abb9b_4d436cb8d7fad8c1b05b38c8375a7637 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:08,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742196_1372 (size=4469) 2024-11-16T05:41:08,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742195_1371 (size=12104) 2024-11-16T05:41:08,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T05:41:08,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:08,509 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T05:41:08,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167c6b56c3c3374acb9b55ca348a89363a_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735666846/Put/seqid=0 2024-11-16T05:41:08,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742197_1373 (size=12154) 2024-11-16T05:41:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T05:41:08,845 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:08,846 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/9dc2b601f478439da9f93463cb81727f is 175, key is test_row_0/A:col10/1731735664595/Put/seqid=0 2024-11-16T05:41:08,856 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/3096e165422647d5a14424badf70bf8f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/3096e165422647d5a14424badf70bf8f 2024-11-16T05:41:08,861 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/B of 4d436cb8d7fad8c1b05b38c8375a7637 into 3096e165422647d5a14424badf70bf8f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:08,861 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:08,861 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/B, priority=13, startTime=1731735668419; duration=0sec 2024-11-16T05:41:08,861 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:08,862 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:B 2024-11-16T05:41:08,862 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:08,865 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:08,865 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/C is initiating minor compaction (all files) 2024-11-16T05:41:08,865 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/C in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:08,865 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/4a27e63849be44bbbb03f2783f77c51d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c29c93541c4a47bbb71f4eda94b42225, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c7b674c0c62e4d01b32627a66459d2fd] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=35.2 K 2024-11-16T05:41:08,866 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a27e63849be44bbbb03f2783f77c51d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731735663901 2024-11-16T05:41:08,866 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting c29c93541c4a47bbb71f4eda94b42225, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731735663949 2024-11-16T05:41:08,866 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting c7b674c0c62e4d01b32627a66459d2fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735664595 2024-11-16T05:41:08,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742198_1374 (size=31058) 2024-11-16T05:41:08,879 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:08,880 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/cecb92ac5cc6462c930a466981c878b5 is 50, key is test_row_0/C:col10/1731735664595/Put/seqid=0 2024-11-16T05:41:08,887 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/9dc2b601f478439da9f93463cb81727f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/9dc2b601f478439da9f93463cb81727f 2024-11-16T05:41:08,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742199_1375 (size=12104) 2024-11-16T05:41:08,893 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/A of 4d436cb8d7fad8c1b05b38c8375a7637 into 9dc2b601f478439da9f93463cb81727f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:08,893 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:08,894 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/A, priority=13, startTime=1731735668419; duration=0sec 2024-11-16T05:41:08,895 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:08,895 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:A 2024-11-16T05:41:08,906 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/cecb92ac5cc6462c930a466981c878b5 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cecb92ac5cc6462c930a466981c878b5 2024-11-16T05:41:08,914 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/C of 4d436cb8d7fad8c1b05b38c8375a7637 into cecb92ac5cc6462c930a466981c878b5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:08,914 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:08,914 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/C, priority=13, startTime=1731735668419; duration=0sec 2024-11-16T05:41:08,914 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:08,914 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:C 2024-11-16T05:41:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,933 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167c6b56c3c3374acb9b55ca348a89363a_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167c6b56c3c3374acb9b55ca348a89363a_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf is 175, key is test_row_0/A:col10/1731735666846/Put/seqid=0 2024-11-16T05:41:08,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742200_1376 (size=30955) 2024-11-16T05:41:08,949 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf 2024-11-16T05:41:08,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/8edc2326bb864097a9c39497284a4e52 is 50, key is test_row_0/B:col10/1731735666846/Put/seqid=0 2024-11-16T05:41:08,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742201_1377 (size=12001) 2024-11-16T05:41:08,968 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/8edc2326bb864097a9c39497284a4e52 2024-11-16T05:41:08,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/5f84df10ac664eba8db54c7f8c51a9c1 is 50, key is test_row_0/C:col10/1731735666846/Put/seqid=0 2024-11-16T05:41:08,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742202_1378 (size=12001) 2024-11-16T05:41:08,981 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/5f84df10ac664eba8db54c7f8c51a9c1 2024-11-16T05:41:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf 2024-11-16T05:41:08,989 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf, entries=150, sequenceid=76, filesize=30.2 K 2024-11-16T05:41:08,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/8edc2326bb864097a9c39497284a4e52 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/8edc2326bb864097a9c39497284a4e52 2024-11-16T05:41:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,993 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/8edc2326bb864097a9c39497284a4e52, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T05:41:08,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/5f84df10ac664eba8db54c7f8c51a9c1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/5f84df10ac664eba8db54c7f8c51a9c1 2024-11-16T05:41:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,998 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/5f84df10ac664eba8db54c7f8c51a9c1, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T05:41:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,999 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 4d436cb8d7fad8c1b05b38c8375a7637 in 490ms, sequenceid=76, compaction requested=false 2024-11-16T05:41:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-16T05:41:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-16T05:41:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-16T05:41:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 950 msec 2024-11-16T05:41:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 954 msec 2024-11-16T05:41:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:09,085 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:09,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:41:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111601b7f3c9c5404575b29e71b6c9601ee5_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735669083/Put/seqid=0 2024-11-16T05:41:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742204_1380 (size=24358) 2024-11-16T05:41:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,104 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111601b7f3c9c5404575b29e71b6c9601ee5_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111601b7f3c9c5404575b29e71b6c9601ee5_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/04beb7d19e214a97b20239052f72369d, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/04beb7d19e214a97b20239052f72369d is 175, key is test_row_0/A:col10/1731735669083/Put/seqid=0 2024-11-16T05:41:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742203_1379 (size=73994) 2024-11-16T05:41:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:09,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735729152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735729151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735729159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735729160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735729160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T05:41:09,188 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:09,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-16T05:41:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T05:41:09,190 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:09,190 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:09,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:09,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735729268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735729268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735729268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735729278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735729278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T05:41:09,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T05:41:09,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:09,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735729478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735729478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735729478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735729488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735729488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T05:41:09,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:09,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T05:41:09,514 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/04beb7d19e214a97b20239052f72369d 2024-11-16T05:41:09,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/5c89b5370a4040038f38d8b941461586 is 50, key is test_row_0/B:col10/1731735669083/Put/seqid=0 2024-11-16T05:41:09,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742205_1381 (size=12001) 2024-11-16T05:41:09,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T05:41:09,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:09,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735729788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735729788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735729789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T05:41:09,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:09,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735729798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735729798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T05:41:09,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/5c89b5370a4040038f38d8b941461586 2024-11-16T05:41:09,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/582c7f4e5d02423ea9344282dadfcb15 is 50, key is test_row_0/C:col10/1731735669083/Put/seqid=0 2024-11-16T05:41:09,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742206_1382 (size=12001) 2024-11-16T05:41:09,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/582c7f4e5d02423ea9344282dadfcb15 2024-11-16T05:41:09,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/04beb7d19e214a97b20239052f72369d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/04beb7d19e214a97b20239052f72369d 2024-11-16T05:41:09,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/04beb7d19e214a97b20239052f72369d, entries=400, sequenceid=90, filesize=72.3 K 2024-11-16T05:41:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/5c89b5370a4040038f38d8b941461586 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5c89b5370a4040038f38d8b941461586 2024-11-16T05:41:09,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T05:41:09,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:09,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:09,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5c89b5370a4040038f38d8b941461586, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T05:41:09,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/582c7f4e5d02423ea9344282dadfcb15 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/582c7f4e5d02423ea9344282dadfcb15 2024-11-16T05:41:09,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/582c7f4e5d02423ea9344282dadfcb15, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T05:41:09,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4d436cb8d7fad8c1b05b38c8375a7637 in 883ms, sequenceid=90, compaction requested=true 2024-11-16T05:41:09,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:09,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:09,968 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:09,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:09,968 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:09,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:09,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:09,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:09,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:09,969 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:09,970 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/B is initiating minor compaction (all files) 2024-11-16T05:41:09,970 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/B in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,970 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/3096e165422647d5a14424badf70bf8f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/8edc2326bb864097a9c39497284a4e52, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5c89b5370a4040038f38d8b941461586] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=35.3 K 2024-11-16T05:41:09,971 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3096e165422647d5a14424badf70bf8f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735664595 2024-11-16T05:41:09,969 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:09,972 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/A is initiating minor compaction (all files) 2024-11-16T05:41:09,972 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/A in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,972 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/9dc2b601f478439da9f93463cb81727f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/04beb7d19e214a97b20239052f72369d] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=132.8 K 2024-11-16T05:41:09,972 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:09,972 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/9dc2b601f478439da9f93463cb81727f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/04beb7d19e214a97b20239052f72369d] 2024-11-16T05:41:09,973 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 8edc2326bb864097a9c39497284a4e52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731735666829 2024-11-16T05:41:09,973 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dc2b601f478439da9f93463cb81727f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735664595 2024-11-16T05:41:09,974 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0be07d3c8eb947c9bb49dc7cd6a6e5bf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731735666829 2024-11-16T05:41:09,974 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c89b5370a4040038f38d8b941461586, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731735669076 2024-11-16T05:41:09,975 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04beb7d19e214a97b20239052f72369d, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731735669063 2024-11-16T05:41:09,985 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#B#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:09,986 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/b9f7061c1c204638ab01cb407d2e1429 is 50, key is test_row_0/B:col10/1731735669083/Put/seqid=0 2024-11-16T05:41:09,990 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:09,993 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411166620116f5ed34c4c8676960cecc08513_4d436cb8d7fad8c1b05b38c8375a7637 store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:09,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742207_1383 (size=12207) 2024-11-16T05:41:09,998 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411166620116f5ed34c4c8676960cecc08513_4d436cb8d7fad8c1b05b38c8375a7637, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:09,998 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166620116f5ed34c4c8676960cecc08513_4d436cb8d7fad8c1b05b38c8375a7637 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:10,013 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/b9f7061c1c204638ab01cb407d2e1429 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/b9f7061c1c204638ab01cb407d2e1429 2024-11-16T05:41:10,022 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/B of 4d436cb8d7fad8c1b05b38c8375a7637 into b9f7061c1c204638ab01cb407d2e1429(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:10,023 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:10,023 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/B, priority=13, startTime=1731735669968; duration=0sec 2024-11-16T05:41:10,023 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:10,023 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:B 2024-11-16T05:41:10,023 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:10,024 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:10,024 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/C is initiating minor compaction (all files) 2024-11-16T05:41:10,024 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/C in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:10,025 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cecb92ac5cc6462c930a466981c878b5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/5f84df10ac664eba8db54c7f8c51a9c1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/582c7f4e5d02423ea9344282dadfcb15] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=35.3 K 2024-11-16T05:41:10,025 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting cecb92ac5cc6462c930a466981c878b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731735664595 2024-11-16T05:41:10,025 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f84df10ac664eba8db54c7f8c51a9c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731735666829 2024-11-16T05:41:10,026 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 582c7f4e5d02423ea9344282dadfcb15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731735669076 2024-11-16T05:41:10,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742208_1384 (size=4469) 2024-11-16T05:41:10,037 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#A#compaction#327 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:10,038 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/dc735fa64b104f8c9395bd2d251e6b6e is 175, key is test_row_0/A:col10/1731735669083/Put/seqid=0 2024-11-16T05:41:10,044 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#C#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:10,044 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/ae23e5ff211f42a3bfbdbcc72acce438 is 50, key is test_row_0/C:col10/1731735669083/Put/seqid=0 2024-11-16T05:41:10,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742209_1385 (size=31161) 2024-11-16T05:41:10,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742210_1386 (size=12207) 2024-11-16T05:41:10,083 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/dc735fa64b104f8c9395bd2d251e6b6e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/dc735fa64b104f8c9395bd2d251e6b6e 2024-11-16T05:41:10,088 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/A of 4d436cb8d7fad8c1b05b38c8375a7637 into dc735fa64b104f8c9395bd2d251e6b6e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:10,088 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:10,088 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/A, priority=13, startTime=1731735669968; duration=0sec 2024-11-16T05:41:10,088 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:10,088 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:A 2024-11-16T05:41:10,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T05:41:10,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:10,108 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T05:41:10,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162ab521019dc3415197026821e5641981_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735669151/Put/seqid=0 2024-11-16T05:41:10,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742211_1387 (size=12154) 2024-11-16T05:41:10,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:10,164 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162ab521019dc3415197026821e5641981_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162ab521019dc3415197026821e5641981_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/5117aecbfcbd4ebea9f0af59e42b67d5, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/5117aecbfcbd4ebea9f0af59e42b67d5 is 175, key is test_row_0/A:col10/1731735669151/Put/seqid=0 2024-11-16T05:41:10,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742212_1388 (size=30955) 2024-11-16T05:41:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:10,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:10,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735730318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735730318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735730320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T05:41:10,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735730327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735730327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735730438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735730448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735730448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,483 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/ae23e5ff211f42a3bfbdbcc72acce438 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ae23e5ff211f42a3bfbdbcc72acce438 2024-11-16T05:41:10,489 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/C of 4d436cb8d7fad8c1b05b38c8375a7637 into ae23e5ff211f42a3bfbdbcc72acce438(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:10,489 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:10,489 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/C, priority=13, startTime=1731735669968; duration=0sec 2024-11-16T05:41:10,489 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:10,489 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:C 2024-11-16T05:41:10,598 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/5117aecbfcbd4ebea9f0af59e42b67d5 2024-11-16T05:41:10,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/2129f41e179848f6aa09908dde3e4442 is 50, key is test_row_0/B:col10/1731735669151/Put/seqid=0 2024-11-16T05:41:10,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742213_1389 (size=12001) 2024-11-16T05:41:10,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735730648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735730658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735730658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735730958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735730968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735730968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,010 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/2129f41e179848f6aa09908dde3e4442 2024-11-16T05:41:11,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/e44e0b738a674893b92e00fe7430671a is 50, key is test_row_0/C:col10/1731735669151/Put/seqid=0 2024-11-16T05:41:11,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742214_1390 (size=12001) 2024-11-16T05:41:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T05:41:11,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735731338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735731338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,341 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:11,341 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,419 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/e44e0b738a674893b92e00fe7430671a 2024-11-16T05:41:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/5117aecbfcbd4ebea9f0af59e42b67d5 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/5117aecbfcbd4ebea9f0af59e42b67d5 2024-11-16T05:41:11,427 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/5117aecbfcbd4ebea9f0af59e42b67d5, entries=150, sequenceid=117, filesize=30.2 K 2024-11-16T05:41:11,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/2129f41e179848f6aa09908dde3e4442 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/2129f41e179848f6aa09908dde3e4442 2024-11-16T05:41:11,437 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/2129f41e179848f6aa09908dde3e4442, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T05:41:11,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/e44e0b738a674893b92e00fe7430671a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/e44e0b738a674893b92e00fe7430671a 2024-11-16T05:41:11,449 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/e44e0b738a674893b92e00fe7430671a, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T05:41:11,450 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1342ms, sequenceid=117, compaction requested=false 2024-11-16T05:41:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-16T05:41:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-16T05:41:11,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-16T05:41:11,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2620 sec 2024-11-16T05:41:11,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.2690 sec 2024-11-16T05:41:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:11,471 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:11,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T05:41:11,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fb0b83cc5a5744c4a6ece653dff20eb0_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735670325/Put/seqid=0 2024-11-16T05:41:11,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742215_1391 (size=14644) 2024-11-16T05:41:11,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735731535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735731542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735731542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735731648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735731658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735731658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735731858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735731868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735731868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:11,901 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:11,905 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fb0b83cc5a5744c4a6ece653dff20eb0_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fb0b83cc5a5744c4a6ece653dff20eb0_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:11,906 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/297f5fb48e69477bb53ca6fb64240beb, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:11,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/297f5fb48e69477bb53ca6fb64240beb is 175, key is test_row_0/A:col10/1731735670325/Put/seqid=0 2024-11-16T05:41:11,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742216_1392 (size=39599) 2024-11-16T05:41:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735732167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:12,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735732178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:12,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735732178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:12,310 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/297f5fb48e69477bb53ca6fb64240beb 2024-11-16T05:41:12,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/d8436a2e52974e97a4069efe84f0989a is 50, key is test_row_0/B:col10/1731735670325/Put/seqid=0 2024-11-16T05:41:12,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742217_1393 (size=12051) 2024-11-16T05:41:12,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735732677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:12,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735732688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:12,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735732688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:12,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/d8436a2e52974e97a4069efe84f0989a 2024-11-16T05:41:12,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/49ab696583374845a1bb980170c633aa is 50, key is test_row_0/C:col10/1731735670325/Put/seqid=0 2024-11-16T05:41:12,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742218_1394 (size=12051) 2024-11-16T05:41:13,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/49ab696583374845a1bb980170c633aa 2024-11-16T05:41:13,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/297f5fb48e69477bb53ca6fb64240beb as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/297f5fb48e69477bb53ca6fb64240beb 2024-11-16T05:41:13,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/297f5fb48e69477bb53ca6fb64240beb, entries=200, sequenceid=130, filesize=38.7 K 2024-11-16T05:41:13,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/d8436a2e52974e97a4069efe84f0989a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8436a2e52974e97a4069efe84f0989a 2024-11-16T05:41:13,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8436a2e52974e97a4069efe84f0989a, entries=150, sequenceid=130, filesize=11.8 K 2024-11-16T05:41:13,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/49ab696583374845a1bb980170c633aa as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/49ab696583374845a1bb980170c633aa 2024-11-16T05:41:13,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/49ab696583374845a1bb980170c633aa, entries=150, sequenceid=130, filesize=11.8 K 2024-11-16T05:41:13,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1676ms, sequenceid=130, compaction requested=true 2024-11-16T05:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:13,147 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:13,147 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:13,148 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:13,148 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101715 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:13,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:13,148 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/A is initiating minor compaction (all files) 2024-11-16T05:41:13,148 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/B is initiating minor compaction (all files) 2024-11-16T05:41:13,148 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/A in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,148 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/B in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,148 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/dc735fa64b104f8c9395bd2d251e6b6e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/5117aecbfcbd4ebea9f0af59e42b67d5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/297f5fb48e69477bb53ca6fb64240beb] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=99.3 K 2024-11-16T05:41:13,148 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/b9f7061c1c204638ab01cb407d2e1429, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/2129f41e179848f6aa09908dde3e4442, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8436a2e52974e97a4069efe84f0989a] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=35.4 K 2024-11-16T05:41:13,148 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,148 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/dc735fa64b104f8c9395bd2d251e6b6e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/5117aecbfcbd4ebea9f0af59e42b67d5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/297f5fb48e69477bb53ca6fb64240beb] 2024-11-16T05:41:13,149 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc735fa64b104f8c9395bd2d251e6b6e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731735669076 2024-11-16T05:41:13,149 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting b9f7061c1c204638ab01cb407d2e1429, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731735669076 2024-11-16T05:41:13,149 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5117aecbfcbd4ebea9f0af59e42b67d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731735669151 2024-11-16T05:41:13,149 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2129f41e179848f6aa09908dde3e4442, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731735669151 2024-11-16T05:41:13,149 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 297f5fb48e69477bb53ca6fb64240beb, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731735670301 2024-11-16T05:41:13,149 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting d8436a2e52974e97a4069efe84f0989a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731735670301 2024-11-16T05:41:13,156 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:13,157 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:13,158 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/ed89415f85214a29b0598fb03bcbb047 is 50, key is test_row_0/B:col10/1731735670325/Put/seqid=0 2024-11-16T05:41:13,158 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e4ca690821794605bf4a6ef2ff19e9ac_4d436cb8d7fad8c1b05b38c8375a7637 store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:13,160 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e4ca690821794605bf4a6ef2ff19e9ac_4d436cb8d7fad8c1b05b38c8375a7637, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:13,160 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e4ca690821794605bf4a6ef2ff19e9ac_4d436cb8d7fad8c1b05b38c8375a7637 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:13,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742219_1395 (size=12359) 2024-11-16T05:41:13,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742220_1396 (size=4469) 2024-11-16T05:41:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T05:41:13,348 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:13,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-16T05:41:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T05:41:13,350 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:13,351 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:13,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:13,363 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:13,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T05:41:13,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dfda4de6dfb843bb9ffc39e0d7417108_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735671541/Put/seqid=0 2024-11-16T05:41:13,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742221_1397 (size=14794) 2024-11-16T05:41:13,378 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:13,381 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dfda4de6dfb843bb9ffc39e0d7417108_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dfda4de6dfb843bb9ffc39e0d7417108_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:13,382 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/2c84ff7ec5984e98ace38cb11afeb315, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:13,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/2c84ff7ec5984e98ace38cb11afeb315 is 175, key is test_row_0/A:col10/1731735671541/Put/seqid=0 2024-11-16T05:41:13,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735733384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:13,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:13,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742222_1398 (size=39749) 2024-11-16T05:41:13,387 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/2c84ff7ec5984e98ace38cb11afeb315 2024-11-16T05:41:13,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/c62e0b5f535e4aa7a2b13b44962ad750 is 50, key is test_row_0/B:col10/1731735671541/Put/seqid=0 2024-11-16T05:41:13,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735733387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:13,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742223_1399 (size=12151) 2024-11-16T05:41:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T05:41:13,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735733487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:13,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T05:41:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735733508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:13,567 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/ed89415f85214a29b0598fb03bcbb047 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/ed89415f85214a29b0598fb03bcbb047 2024-11-16T05:41:13,569 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#A#compaction#336 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:13,569 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/59b7bface58845f9b887b5643f72e643 is 175, key is test_row_0/A:col10/1731735670325/Put/seqid=0 2024-11-16T05:41:13,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742224_1400 (size=31313) 2024-11-16T05:41:13,573 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/B of 4d436cb8d7fad8c1b05b38c8375a7637 into ed89415f85214a29b0598fb03bcbb047(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:13,573 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:13,573 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/B, priority=13, startTime=1731735673147; duration=0sec 2024-11-16T05:41:13,574 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:13,574 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:B 2024-11-16T05:41:13,575 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:13,577 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:13,577 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/C is initiating minor compaction (all files) 2024-11-16T05:41:13,577 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/C in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,577 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ae23e5ff211f42a3bfbdbcc72acce438, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/e44e0b738a674893b92e00fe7430671a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/49ab696583374845a1bb980170c633aa] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=35.4 K 2024-11-16T05:41:13,578 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ae23e5ff211f42a3bfbdbcc72acce438, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731735669076 2024-11-16T05:41:13,578 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e44e0b738a674893b92e00fe7430671a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731735669151 2024-11-16T05:41:13,578 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 49ab696583374845a1bb980170c633aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731735670301 2024-11-16T05:41:13,580 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/59b7bface58845f9b887b5643f72e643 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/59b7bface58845f9b887b5643f72e643 2024-11-16T05:41:13,584 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/A of 4d436cb8d7fad8c1b05b38c8375a7637 into 59b7bface58845f9b887b5643f72e643(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:13,584 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:13,584 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/A, priority=13, startTime=1731735673147; duration=0sec 2024-11-16T05:41:13,584 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:13,584 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:A 2024-11-16T05:41:13,585 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:13,585 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/29ce4d6506394d27bd85007efbf84d09 is 50, key is test_row_0/C:col10/1731735670325/Put/seqid=0 2024-11-16T05:41:13,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742225_1401 (size=12359) 2024-11-16T05:41:13,593 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/29ce4d6506394d27bd85007efbf84d09 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/29ce4d6506394d27bd85007efbf84d09 2024-11-16T05:41:13,597 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/C of 4d436cb8d7fad8c1b05b38c8375a7637 into 29ce4d6506394d27bd85007efbf84d09(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:13,597 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:13,597 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/C, priority=13, startTime=1731735673147; duration=0sec 2024-11-16T05:41:13,597 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:13,597 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:C 2024-11-16T05:41:13,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T05:41:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T05:41:13,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735733688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:13,691 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:13,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735733698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:13,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735733698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:13,701 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:13,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735733708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:13,712 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:13,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:13,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735733718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:13,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/c62e0b5f535e4aa7a2b13b44962ad750 2024-11-16T05:41:13,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T05:41:13,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:13,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/986509a4f50c4515898d4314f24d62e1 is 50, key is test_row_0/C:col10/1731735671541/Put/seqid=0 2024-11-16T05:41:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:13,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742226_1402 (size=12151) 2024-11-16T05:41:13,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/986509a4f50c4515898d4314f24d62e1 2024-11-16T05:41:13,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/2c84ff7ec5984e98ace38cb11afeb315 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2c84ff7ec5984e98ace38cb11afeb315 2024-11-16T05:41:13,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2c84ff7ec5984e98ace38cb11afeb315, entries=200, sequenceid=156, filesize=38.8 K 2024-11-16T05:41:13,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/c62e0b5f535e4aa7a2b13b44962ad750 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c62e0b5f535e4aa7a2b13b44962ad750 2024-11-16T05:41:13,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c62e0b5f535e4aa7a2b13b44962ad750, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T05:41:13,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/986509a4f50c4515898d4314f24d62e1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/986509a4f50c4515898d4314f24d62e1 2024-11-16T05:41:13,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/986509a4f50c4515898d4314f24d62e1, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T05:41:13,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 4d436cb8d7fad8c1b05b38c8375a7637 in 465ms, sequenceid=156, compaction requested=false 2024-11-16T05:41:13,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:13,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T05:41:13,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:13,961 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T05:41:13,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111686fbe1289a1f46cebe56f6c759dd88d5_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735673384/Put/seqid=0 2024-11-16T05:41:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T05:41:13,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742227_1403 (size=12304) 2024-11-16T05:41:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:14,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:14,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735734086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:14,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735734089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:14,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735734198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:14,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735734198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:14,385 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111686fbe1289a1f46cebe56f6c759dd88d5_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111686fbe1289a1f46cebe56f6c759dd88d5_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/1d5588456cf74707a140ac56f7ca9c81, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/1d5588456cf74707a140ac56f7ca9c81 is 175, key is test_row_0/A:col10/1731735673384/Put/seqid=0 2024-11-16T05:41:14,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742228_1404 (size=31105) 2024-11-16T05:41:14,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735734408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:14,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:14,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735734408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T05:41:14,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735734717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:14,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735734718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:14,791 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/1d5588456cf74707a140ac56f7ca9c81 2024-11-16T05:41:14,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/1152d97b11f24f2ca6a91cce51c233f9 is 50, key is test_row_0/B:col10/1731735673384/Put/seqid=0 2024-11-16T05:41:14,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742229_1405 (size=12151) 2024-11-16T05:41:15,203 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/1152d97b11f24f2ca6a91cce51c233f9 2024-11-16T05:41:15,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/1466e08e589144eea03b79a58d22c1f3 is 50, key is test_row_0/C:col10/1731735673384/Put/seqid=0 2024-11-16T05:41:15,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742230_1406 (size=12151) 2024-11-16T05:41:15,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735735228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735735228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T05:41:15,614 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/1466e08e589144eea03b79a58d22c1f3 2024-11-16T05:41:15,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/1d5588456cf74707a140ac56f7ca9c81 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/1d5588456cf74707a140ac56f7ca9c81 2024-11-16T05:41:15,620 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/1d5588456cf74707a140ac56f7ca9c81, entries=150, sequenceid=169, filesize=30.4 K 2024-11-16T05:41:15,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/1152d97b11f24f2ca6a91cce51c233f9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/1152d97b11f24f2ca6a91cce51c233f9 2024-11-16T05:41:15,624 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/1152d97b11f24f2ca6a91cce51c233f9, entries=150, sequenceid=169, filesize=11.9 K 2024-11-16T05:41:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/1466e08e589144eea03b79a58d22c1f3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/1466e08e589144eea03b79a58d22c1f3 2024-11-16T05:41:15,628 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/1466e08e589144eea03b79a58d22c1f3, entries=150, sequenceid=169, filesize=11.9 K 2024-11-16T05:41:15,629 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1668ms, sequenceid=169, compaction requested=true 2024-11-16T05:41:15,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:15,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:15,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-16T05:41:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-16T05:41:15,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-16T05:41:15,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2790 sec 2024-11-16T05:41:15,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.2840 sec 2024-11-16T05:41:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:15,713 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:15,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T05:41:15,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111652f7223b0a2b404c8362582be1eb7313_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735675712/Put/seqid=0 2024-11-16T05:41:15,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742231_1407 (size=17284) 2024-11-16T05:41:15,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735735723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:15,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735735728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:15,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735735729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:15,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735735837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:15,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735735838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:15,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735735838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735736048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735736048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735736049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,124 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:16,127 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111652f7223b0a2b404c8362582be1eb7313_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111652f7223b0a2b404c8362582be1eb7313_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:16,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/e803cf73e4cf4817a54121099b1f48cf, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:16,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/e803cf73e4cf4817a54121099b1f48cf is 175, key is test_row_0/A:col10/1731735675712/Put/seqid=0 2024-11-16T05:41:16,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742232_1408 (size=48389) 2024-11-16T05:41:16,138 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/e803cf73e4cf4817a54121099b1f48cf 2024-11-16T05:41:16,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/6667dabb53574eceb41bc37ba8a4e444 is 50, key is test_row_0/B:col10/1731735675712/Put/seqid=0 2024-11-16T05:41:16,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742233_1409 (size=12151) 2024-11-16T05:41:16,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/6667dabb53574eceb41bc37ba8a4e444 2024-11-16T05:41:16,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/b5c9ac1bbf9a4ccb92bcd2e05955676a is 50, key is test_row_0/C:col10/1731735675712/Put/seqid=0 2024-11-16T05:41:16,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742234_1410 (size=12151) 2024-11-16T05:41:16,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/b5c9ac1bbf9a4ccb92bcd2e05955676a 2024-11-16T05:41:16,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/e803cf73e4cf4817a54121099b1f48cf as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/e803cf73e4cf4817a54121099b1f48cf 2024-11-16T05:41:16,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/e803cf73e4cf4817a54121099b1f48cf, entries=250, sequenceid=197, filesize=47.3 K 2024-11-16T05:41:16,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/6667dabb53574eceb41bc37ba8a4e444 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/6667dabb53574eceb41bc37ba8a4e444 2024-11-16T05:41:16,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/6667dabb53574eceb41bc37ba8a4e444, entries=150, sequenceid=197, filesize=11.9 K 2024-11-16T05:41:16,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/b5c9ac1bbf9a4ccb92bcd2e05955676a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/b5c9ac1bbf9a4ccb92bcd2e05955676a 2024-11-16T05:41:16,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/b5c9ac1bbf9a4ccb92bcd2e05955676a, entries=150, sequenceid=197, filesize=11.9 K 2024-11-16T05:41:16,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 4d436cb8d7fad8c1b05b38c8375a7637 in 474ms, sequenceid=197, compaction requested=true 2024-11-16T05:41:16,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:16,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:16,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:16,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:16,188 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:41:16,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:16,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:16,188 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:41:16,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:16,189 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150556 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:41:16,189 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:41:16,189 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/A is initiating minor compaction (all files) 2024-11-16T05:41:16,189 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/B is initiating minor compaction (all files) 2024-11-16T05:41:16,189 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/A in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:16,189 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/B in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:16,189 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/ed89415f85214a29b0598fb03bcbb047, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c62e0b5f535e4aa7a2b13b44962ad750, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/1152d97b11f24f2ca6a91cce51c233f9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/6667dabb53574eceb41bc37ba8a4e444] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=47.7 K 2024-11-16T05:41:16,189 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/59b7bface58845f9b887b5643f72e643, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2c84ff7ec5984e98ace38cb11afeb315, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/1d5588456cf74707a140ac56f7ca9c81, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/e803cf73e4cf4817a54121099b1f48cf] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=147.0 K 2024-11-16T05:41:16,189 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:16,189 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/59b7bface58845f9b887b5643f72e643, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2c84ff7ec5984e98ace38cb11afeb315, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/1d5588456cf74707a140ac56f7ca9c81, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/e803cf73e4cf4817a54121099b1f48cf] 2024-11-16T05:41:16,189 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ed89415f85214a29b0598fb03bcbb047, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731735670301 2024-11-16T05:41:16,189 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59b7bface58845f9b887b5643f72e643, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731735670301 2024-11-16T05:41:16,189 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting c62e0b5f535e4aa7a2b13b44962ad750, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731735671533 2024-11-16T05:41:16,189 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c84ff7ec5984e98ace38cb11afeb315, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731735671533 2024-11-16T05:41:16,190 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1152d97b11f24f2ca6a91cce51c233f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731735673374 2024-11-16T05:41:16,190 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d5588456cf74707a140ac56f7ca9c81, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731735673374 2024-11-16T05:41:16,190 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6667dabb53574eceb41bc37ba8a4e444, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731735674088 2024-11-16T05:41:16,190 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e803cf73e4cf4817a54121099b1f48cf, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731735674080 2024-11-16T05:41:16,198 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:16,199 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411165454ad3bf7f64e338f65f11e47d9ee3b_4d436cb8d7fad8c1b05b38c8375a7637 store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:16,200 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#B#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:16,200 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/d8e4580293334a009586aa471401f221 is 50, key is test_row_0/B:col10/1731735675712/Put/seqid=0 2024-11-16T05:41:16,204 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411165454ad3bf7f64e338f65f11e47d9ee3b_4d436cb8d7fad8c1b05b38c8375a7637, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:16,204 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165454ad3bf7f64e338f65f11e47d9ee3b_4d436cb8d7fad8c1b05b38c8375a7637 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:16,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742235_1411 (size=12595) 2024-11-16T05:41:16,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742236_1412 (size=4469) 2024-11-16T05:41:16,209 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#A#compaction#348 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:16,210 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/2acda43d2eed4e858eb2808f3b9d9854 is 175, key is test_row_0/A:col10/1731735675712/Put/seqid=0 2024-11-16T05:41:16,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742237_1413 (size=31549) 2024-11-16T05:41:16,220 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/2acda43d2eed4e858eb2808f3b9d9854 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2acda43d2eed4e858eb2808f3b9d9854 2024-11-16T05:41:16,223 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/A of 4d436cb8d7fad8c1b05b38c8375a7637 into 2acda43d2eed4e858eb2808f3b9d9854(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:16,223 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:16,223 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/A, priority=12, startTime=1731735676188; duration=0sec 2024-11-16T05:41:16,223 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:16,224 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:A 2024-11-16T05:41:16,224 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:41:16,225 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:41:16,225 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/C is initiating minor compaction (all files) 2024-11-16T05:41:16,225 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/C in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:16,225 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/29ce4d6506394d27bd85007efbf84d09, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/986509a4f50c4515898d4314f24d62e1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/1466e08e589144eea03b79a58d22c1f3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/b5c9ac1bbf9a4ccb92bcd2e05955676a] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=47.7 K 2024-11-16T05:41:16,225 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ce4d6506394d27bd85007efbf84d09, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731735670301 2024-11-16T05:41:16,225 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 986509a4f50c4515898d4314f24d62e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731735671533 2024-11-16T05:41:16,225 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1466e08e589144eea03b79a58d22c1f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731735673374 2024-11-16T05:41:16,226 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5c9ac1bbf9a4ccb92bcd2e05955676a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731735674088 2024-11-16T05:41:16,232 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#C#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:16,232 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/cface2354c6d4173adf75b26a0dd8188 is 50, key is test_row_0/C:col10/1731735675712/Put/seqid=0 2024-11-16T05:41:16,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742238_1414 (size=12595) 2024-11-16T05:41:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:16,254 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:16,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:41:16,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161517bdcb68224ad8b0bc91cc147edc96_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735676251/Put/seqid=0 2024-11-16T05:41:16,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742239_1415 (size=14794) 2024-11-16T05:41:16,265 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:16,268 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161517bdcb68224ad8b0bc91cc147edc96_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161517bdcb68224ad8b0bc91cc147edc96_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:16,268 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/ea3800a39948488cbe72f33a9e52dff7, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:16,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/ea3800a39948488cbe72f33a9e52dff7 is 175, key is test_row_0/A:col10/1731735676251/Put/seqid=0 2024-11-16T05:41:16,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742240_1416 (size=39749) 2024-11-16T05:41:16,273 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/ea3800a39948488cbe72f33a9e52dff7 2024-11-16T05:41:16,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/c92d7dfbbddc4ba3a1042d975629a981 is 50, key is test_row_0/B:col10/1731735676251/Put/seqid=0 2024-11-16T05:41:16,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742241_1417 (size=12151) 2024-11-16T05:41:16,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735736337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735736341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735736358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735736358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735736358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735736448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735736458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,611 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/d8e4580293334a009586aa471401f221 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8e4580293334a009586aa471401f221 2024-11-16T05:41:16,615 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/B of 4d436cb8d7fad8c1b05b38c8375a7637 into d8e4580293334a009586aa471401f221(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:16,615 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:16,615 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/B, priority=12, startTime=1731735676188; duration=0sec 2024-11-16T05:41:16,615 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:16,615 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:B 2024-11-16T05:41:16,647 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/cface2354c6d4173adf75b26a0dd8188 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cface2354c6d4173adf75b26a0dd8188 2024-11-16T05:41:16,651 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/C of 4d436cb8d7fad8c1b05b38c8375a7637 into cface2354c6d4173adf75b26a0dd8188(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:16,651 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:16,651 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/C, priority=12, startTime=1731735676188; duration=0sec 2024-11-16T05:41:16,651 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:16,651 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:C 2024-11-16T05:41:16,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735736658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735736668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/c92d7dfbbddc4ba3a1042d975629a981 2024-11-16T05:41:16,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/f4bd030552fd43b2a42903fb9034f8fa is 50, key is test_row_0/C:col10/1731735676251/Put/seqid=0 2024-11-16T05:41:16,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742242_1418 (size=12151) 2024-11-16T05:41:16,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/f4bd030552fd43b2a42903fb9034f8fa 2024-11-16T05:41:16,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/ea3800a39948488cbe72f33a9e52dff7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ea3800a39948488cbe72f33a9e52dff7 2024-11-16T05:41:16,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ea3800a39948488cbe72f33a9e52dff7, entries=200, sequenceid=209, filesize=38.8 K 2024-11-16T05:41:16,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/c92d7dfbbddc4ba3a1042d975629a981 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c92d7dfbbddc4ba3a1042d975629a981 2024-11-16T05:41:16,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c92d7dfbbddc4ba3a1042d975629a981, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T05:41:16,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/f4bd030552fd43b2a42903fb9034f8fa as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/f4bd030552fd43b2a42903fb9034f8fa 2024-11-16T05:41:16,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/f4bd030552fd43b2a42903fb9034f8fa, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T05:41:16,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4d436cb8d7fad8c1b05b38c8375a7637 in 458ms, sequenceid=209, compaction requested=false 2024-11-16T05:41:16,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:16,873 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:16,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T05:41:16,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b3cd0ef7f8db42ca9d824c7e0d2448aa_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735676339/Put/seqid=0 2024-11-16T05:41:16,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742243_1419 (size=14794) 2024-11-16T05:41:16,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735736888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735736896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735736898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735736968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:16,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735736978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735737008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735737008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735737017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735737218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735737218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735737228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,285 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:17,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b3cd0ef7f8db42ca9d824c7e0d2448aa_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b3cd0ef7f8db42ca9d824c7e0d2448aa_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:17,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/2b9d11e4823d422bb56bf0dd33abcb54, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:17,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/2b9d11e4823d422bb56bf0dd33abcb54 is 175, key is test_row_0/A:col10/1731735676339/Put/seqid=0 2024-11-16T05:41:17,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742244_1420 (size=39749) 2024-11-16T05:41:17,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735737488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735737497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T05:41:17,508 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:17,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-16T05:41:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:41:17,510 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:17,510 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:17,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:17,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735737528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735737528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735737538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:41:17,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:41:17,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:17,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:17,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:17,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:17,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:17,695 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/2b9d11e4823d422bb56bf0dd33abcb54 2024-11-16T05:41:17,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/5b37de0fc0a144f0ac7200dda3abf2f6 is 50, key is test_row_0/B:col10/1731735676339/Put/seqid=0 2024-11-16T05:41:17,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742245_1421 (size=12151) 2024-11-16T05:41:17,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:41:17,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:17,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:17,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:17,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:17,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:41:17,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:41:17,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:17,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:17,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:17,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:17,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:18,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735738038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:18,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735738038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:18,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735738048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:18,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/5b37de0fc0a144f0ac7200dda3abf2f6 2024-11-16T05:41:18,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/440fbcadc13c4e409856dcfe0f29a425 is 50, key is test_row_0/C:col10/1731735676339/Put/seqid=0 2024-11-16T05:41:18,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742246_1422 (size=12151) 2024-11-16T05:41:18,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/440fbcadc13c4e409856dcfe0f29a425 2024-11-16T05:41:18,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/2b9d11e4823d422bb56bf0dd33abcb54 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2b9d11e4823d422bb56bf0dd33abcb54 2024-11-16T05:41:18,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:41:18,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:18,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:18,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:18,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:18,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:18,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2b9d11e4823d422bb56bf0dd33abcb54, entries=200, sequenceid=237, filesize=38.8 K 2024-11-16T05:41:18,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/5b37de0fc0a144f0ac7200dda3abf2f6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5b37de0fc0a144f0ac7200dda3abf2f6 2024-11-16T05:41:18,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5b37de0fc0a144f0ac7200dda3abf2f6, entries=150, sequenceid=237, filesize=11.9 K 2024-11-16T05:41:18,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/440fbcadc13c4e409856dcfe0f29a425 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/440fbcadc13c4e409856dcfe0f29a425 2024-11-16T05:41:18,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/440fbcadc13c4e409856dcfe0f29a425, entries=150, sequenceid=237, filesize=11.9 K 2024-11-16T05:41:18,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1258ms, sequenceid=237, compaction requested=true 2024-11-16T05:41:18,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:18,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:18,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:18,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:18,131 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:18,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:18,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:18,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:41:18,132 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:18,132 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:18,132 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/A is initiating minor compaction (all files) 2024-11-16T05:41:18,133 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/A in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:18,133 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2acda43d2eed4e858eb2808f3b9d9854, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ea3800a39948488cbe72f33a9e52dff7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2b9d11e4823d422bb56bf0dd33abcb54] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=108.4 K 2024-11-16T05:41:18,133 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:18,133 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2acda43d2eed4e858eb2808f3b9d9854, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ea3800a39948488cbe72f33a9e52dff7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2b9d11e4823d422bb56bf0dd33abcb54] 2024-11-16T05:41:18,133 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:18,133 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/B is initiating minor compaction (all files) 2024-11-16T05:41:18,133 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2acda43d2eed4e858eb2808f3b9d9854, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731735674088 2024-11-16T05:41:18,133 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/B in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:18,133 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8e4580293334a009586aa471401f221, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c92d7dfbbddc4ba3a1042d975629a981, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5b37de0fc0a144f0ac7200dda3abf2f6] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=36.0 K 2024-11-16T05:41:18,133 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3800a39948488cbe72f33a9e52dff7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731735675716 2024-11-16T05:41:18,133 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e4580293334a009586aa471401f221, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731735674088 2024-11-16T05:41:18,134 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c92d7dfbbddc4ba3a1042d975629a981, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731735675716 2024-11-16T05:41:18,134 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9d11e4823d422bb56bf0dd33abcb54, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731735676315 2024-11-16T05:41:18,134 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b37de0fc0a144f0ac7200dda3abf2f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731735676315 2024-11-16T05:41:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:41:18,140 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:18,141 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#B#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:18,142 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/3636d232af1343d9bebff6ac78c61d16 is 50, key is test_row_0/B:col10/1731735676339/Put/seqid=0 2024-11-16T05:41:18,142 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411168588fe4fa825436b8410f4c98acb7c08_4d436cb8d7fad8c1b05b38c8375a7637 store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:18,144 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411168588fe4fa825436b8410f4c98acb7c08_4d436cb8d7fad8c1b05b38c8375a7637, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:18,144 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168588fe4fa825436b8410f4c98acb7c08_4d436cb8d7fad8c1b05b38c8375a7637 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:18,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742248_1424 (size=4469) 2024-11-16T05:41:18,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742247_1423 (size=12697) 2024-11-16T05:41:18,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:41:18,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:18,274 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T05:41:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162a16ce837d824683bfc8be8b1dc75bf8_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735676895/Put/seqid=0 2024-11-16T05:41:18,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742249_1425 (size=12304) 2024-11-16T05:41:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:18,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:18,549 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#A#compaction#357 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:18,550 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/731e58ab729f4f2c98b0c373644fc933 is 175, key is test_row_0/A:col10/1731735676339/Put/seqid=0 2024-11-16T05:41:18,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742250_1426 (size=31651) 2024-11-16T05:41:18,566 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/3636d232af1343d9bebff6ac78c61d16 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/3636d232af1343d9bebff6ac78c61d16 2024-11-16T05:41:18,570 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/B of 4d436cb8d7fad8c1b05b38c8375a7637 into 3636d232af1343d9bebff6ac78c61d16(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:18,570 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:18,570 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/B, priority=13, startTime=1731735678131; duration=0sec 2024-11-16T05:41:18,570 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:18,570 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:B 2024-11-16T05:41:18,571 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:18,571 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:18,571 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/C is initiating minor compaction (all files) 2024-11-16T05:41:18,571 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/C in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:18,572 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cface2354c6d4173adf75b26a0dd8188, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/f4bd030552fd43b2a42903fb9034f8fa, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/440fbcadc13c4e409856dcfe0f29a425] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=36.0 K 2024-11-16T05:41:18,572 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting cface2354c6d4173adf75b26a0dd8188, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731735674088 2024-11-16T05:41:18,572 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4bd030552fd43b2a42903fb9034f8fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731735675716 2024-11-16T05:41:18,572 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 440fbcadc13c4e409856dcfe0f29a425, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731735676315 2024-11-16T05:41:18,578 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#C#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:18,578 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/3cb81c3a6d824b4e8bd1cf6f6ca6962d is 50, key is test_row_0/C:col10/1731735676339/Put/seqid=0 2024-11-16T05:41:18,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742251_1427 (size=12697) 2024-11-16T05:41:18,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735738629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:18,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735738630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:41:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:18,688 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162a16ce837d824683bfc8be8b1dc75bf8_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162a16ce837d824683bfc8be8b1dc75bf8_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/c63628a00c6346cfacd8923bfa39797b, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/c63628a00c6346cfacd8923bfa39797b is 175, key is test_row_0/A:col10/1731735676895/Put/seqid=0 2024-11-16T05:41:18,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742252_1428 (size=31105) 2024-11-16T05:41:18,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735738738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:18,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735738748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:18,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735738948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:18,957 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/731e58ab729f4f2c98b0c373644fc933 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/731e58ab729f4f2c98b0c373644fc933 2024-11-16T05:41:18,961 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/A of 4d436cb8d7fad8c1b05b38c8375a7637 into 731e58ab729f4f2c98b0c373644fc933(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:18,961 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:18,961 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/A, priority=13, startTime=1731735678131; duration=0sec 2024-11-16T05:41:18,961 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:18,961 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:A 2024-11-16T05:41:18,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735738958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:18,986 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/3cb81c3a6d824b4e8bd1cf6f6ca6962d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3cb81c3a6d824b4e8bd1cf6f6ca6962d 2024-11-16T05:41:18,990 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/C of 4d436cb8d7fad8c1b05b38c8375a7637 into 3cb81c3a6d824b4e8bd1cf6f6ca6962d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:18,990 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:18,990 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/C, priority=13, startTime=1731735678131; duration=0sec 2024-11-16T05:41:18,990 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:18,990 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:C 2024-11-16T05:41:19,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735739048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:19,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735739048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:19,052 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:19,052 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:19,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735739058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:19,065 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:19,098 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/c63628a00c6346cfacd8923bfa39797b 2024-11-16T05:41:19,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/7ac0e380cf7f43cc8011c3a1888ca164 is 50, key is test_row_0/B:col10/1731735676895/Put/seqid=0 2024-11-16T05:41:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742253_1429 (size=12151) 2024-11-16T05:41:19,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735739258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:19,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735739268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:19,510 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/7ac0e380cf7f43cc8011c3a1888ca164 2024-11-16T05:41:19,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/a9109d77d8ca42f6a982e653ed4a6fa8 is 50, key is test_row_0/C:col10/1731735676895/Put/seqid=0 2024-11-16T05:41:19,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742254_1430 (size=12151) 2024-11-16T05:41:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:41:19,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735739768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:19,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735739788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:19,918 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/a9109d77d8ca42f6a982e653ed4a6fa8 2024-11-16T05:41:19,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/c63628a00c6346cfacd8923bfa39797b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c63628a00c6346cfacd8923bfa39797b 2024-11-16T05:41:19,924 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c63628a00c6346cfacd8923bfa39797b, entries=150, sequenceid=247, filesize=30.4 K 2024-11-16T05:41:19,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/7ac0e380cf7f43cc8011c3a1888ca164 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/7ac0e380cf7f43cc8011c3a1888ca164 2024-11-16T05:41:19,928 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/7ac0e380cf7f43cc8011c3a1888ca164, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T05:41:19,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/a9109d77d8ca42f6a982e653ed4a6fa8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/a9109d77d8ca42f6a982e653ed4a6fa8 2024-11-16T05:41:19,931 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/a9109d77d8ca42f6a982e653ed4a6fa8, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T05:41:19,932 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1658ms, sequenceid=247, compaction requested=false 2024-11-16T05:41:19,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:19,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:19,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-16T05:41:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-16T05:41:19,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-16T05:41:19,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4230 sec 2024-11-16T05:41:19,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.4270 sec 2024-11-16T05:41:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:20,791 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:20,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T05:41:20,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166176a4ec3af44f2a99517cbe7d899b2f_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735678628/Put/seqid=0 2024-11-16T05:41:20,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742255_1431 (size=14994) 2024-11-16T05:41:20,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735740812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:20,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735740818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:20,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735740928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:20,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735740928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:21,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735741068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:21,073 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:21,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:21,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38576 deadline: 1731735741078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:21,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735741078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:21,081 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:21,081 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:21,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735741138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735741138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:21,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:21,204 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166176a4ec3af44f2a99517cbe7d899b2f_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166176a4ec3af44f2a99517cbe7d899b2f_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:21,205 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/ada14fca885f4ca7998225f64586db5a, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:21,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/ada14fca885f4ca7998225f64586db5a is 175, key is test_row_0/A:col10/1731735678628/Put/seqid=0 2024-11-16T05:41:21,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742256_1432 (size=39949) 2024-11-16T05:41:21,211 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/ada14fca885f4ca7998225f64586db5a 2024-11-16T05:41:21,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/27b8ce1e432a4ab2bafe3beaf961b325 is 50, key is test_row_0/B:col10/1731735678628/Put/seqid=0 2024-11-16T05:41:21,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742257_1433 (size=12301) 2024-11-16T05:41:21,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735741448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:21,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735741448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:21,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/27b8ce1e432a4ab2bafe3beaf961b325 2024-11-16T05:41:21,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/ba12af656c7840dc860de76f3dba114d is 50, key is test_row_0/C:col10/1731735678628/Put/seqid=0 2024-11-16T05:41:21,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742258_1434 (size=12301) 2024-11-16T05:41:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:41:21,668 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:21,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-16T05:41:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:41:21,670 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:21,671 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:21,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:41:21,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T05:41:21,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:21,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:21,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:21,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:21,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735741958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:21,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735741967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:21,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T05:41:21,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:21,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:21,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:21,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:21,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:41:22,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/ba12af656c7840dc860de76f3dba114d 2024-11-16T05:41:22,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/ada14fca885f4ca7998225f64586db5a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ada14fca885f4ca7998225f64586db5a 2024-11-16T05:41:22,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ada14fca885f4ca7998225f64586db5a, entries=200, sequenceid=278, filesize=39.0 K 2024-11-16T05:41:22,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/27b8ce1e432a4ab2bafe3beaf961b325 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/27b8ce1e432a4ab2bafe3beaf961b325 2024-11-16T05:41:22,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/27b8ce1e432a4ab2bafe3beaf961b325, entries=150, sequenceid=278, filesize=12.0 K 2024-11-16T05:41:22,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/ba12af656c7840dc860de76f3dba114d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ba12af656c7840dc860de76f3dba114d 2024-11-16T05:41:22,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ba12af656c7840dc860de76f3dba114d, entries=150, sequenceid=278, filesize=12.0 K 2024-11-16T05:41:22,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1259ms, sequenceid=278, compaction requested=true 2024-11-16T05:41:22,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:22,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:22,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:22,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:22,051 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:22,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:22,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:22,051 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:22,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:22,051 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:22,051 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:22,051 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/A is initiating minor compaction (all files) 2024-11-16T05:41:22,051 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/B is initiating minor compaction (all files) 2024-11-16T05:41:22,051 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/A in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:22,051 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/B in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:22,051 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/3636d232af1343d9bebff6ac78c61d16, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/7ac0e380cf7f43cc8011c3a1888ca164, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/27b8ce1e432a4ab2bafe3beaf961b325] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=36.3 K 2024-11-16T05:41:22,051 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/731e58ab729f4f2c98b0c373644fc933, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c63628a00c6346cfacd8923bfa39797b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ada14fca885f4ca7998225f64586db5a] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=100.3 K 2024-11-16T05:41:22,052 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:22,052 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/731e58ab729f4f2c98b0c373644fc933, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c63628a00c6346cfacd8923bfa39797b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ada14fca885f4ca7998225f64586db5a] 2024-11-16T05:41:22,052 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3636d232af1343d9bebff6ac78c61d16, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731735676315 2024-11-16T05:41:22,052 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731e58ab729f4f2c98b0c373644fc933, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731735676315 2024-11-16T05:41:22,052 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac0e380cf7f43cc8011c3a1888ca164, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731735676885 2024-11-16T05:41:22,052 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c63628a00c6346cfacd8923bfa39797b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731735676885 2024-11-16T05:41:22,052 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b8ce1e432a4ab2bafe3beaf961b325, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731735678608 2024-11-16T05:41:22,052 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ada14fca885f4ca7998225f64586db5a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731735678608 2024-11-16T05:41:22,057 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:22,059 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:22,059 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111640a8ac8a0d504d66a7d6df5e52a9fa1d_4d436cb8d7fad8c1b05b38c8375a7637 store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:22,059 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/40a4924a0cfe4a5eb871615a7b84805e is 50, key is test_row_0/B:col10/1731735678628/Put/seqid=0 2024-11-16T05:41:22,061 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111640a8ac8a0d504d66a7d6df5e52a9fa1d_4d436cb8d7fad8c1b05b38c8375a7637, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:22,061 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111640a8ac8a0d504d66a7d6df5e52a9fa1d_4d436cb8d7fad8c1b05b38c8375a7637 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:22,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742260_1436 (size=4469) 2024-11-16T05:41:22,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742259_1435 (size=12949) 2024-11-16T05:41:22,072 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/40a4924a0cfe4a5eb871615a7b84805e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/40a4924a0cfe4a5eb871615a7b84805e 2024-11-16T05:41:22,076 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/B of 4d436cb8d7fad8c1b05b38c8375a7637 into 40a4924a0cfe4a5eb871615a7b84805e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:22,076 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:22,076 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/B, priority=13, startTime=1731735682051; duration=0sec 2024-11-16T05:41:22,076 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:22,076 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:B 2024-11-16T05:41:22,076 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:22,077 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:22,077 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/C is initiating minor compaction (all files) 2024-11-16T05:41:22,077 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/C in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:22,077 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3cb81c3a6d824b4e8bd1cf6f6ca6962d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/a9109d77d8ca42f6a982e653ed4a6fa8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ba12af656c7840dc860de76f3dba114d] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=36.3 K 2024-11-16T05:41:22,077 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cb81c3a6d824b4e8bd1cf6f6ca6962d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731735676315 2024-11-16T05:41:22,077 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting a9109d77d8ca42f6a982e653ed4a6fa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731735676885 2024-11-16T05:41:22,078 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ba12af656c7840dc860de76f3dba114d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731735678608 2024-11-16T05:41:22,082 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:22,083 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/ef6cf7a6499443c0bc99737b84989f64 is 50, key is test_row_0/C:col10/1731735678628/Put/seqid=0 2024-11-16T05:41:22,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742261_1437 (size=12949) 2024-11-16T05:41:22,091 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/ef6cf7a6499443c0bc99737b84989f64 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ef6cf7a6499443c0bc99737b84989f64 2024-11-16T05:41:22,097 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/C of 4d436cb8d7fad8c1b05b38c8375a7637 into ef6cf7a6499443c0bc99737b84989f64(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:22,097 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:22,097 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/C, priority=13, startTime=1731735682051; duration=0sec 2024-11-16T05:41:22,097 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:22,097 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:C 2024-11-16T05:41:22,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T05:41:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:22,128 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T05:41:22,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fdde4f6fefc0493abcdc72811f5f41f6_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735680811/Put/seqid=0 2024-11-16T05:41:22,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742262_1438 (size=12454) 2024-11-16T05:41:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:41:22,467 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#A#compaction#365 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:22,468 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/b55cae63336d4679ac3da41e8c21bf08 is 175, key is test_row_0/A:col10/1731735678628/Put/seqid=0 2024-11-16T05:41:22,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742263_1439 (size=31903) 2024-11-16T05:41:22,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:22,542 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fdde4f6fefc0493abcdc72811f5f41f6_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fdde4f6fefc0493abcdc72811f5f41f6_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/c2935ac895a440cc811fddab68f93dc5, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/c2935ac895a440cc811fddab68f93dc5 is 175, key is test_row_0/A:col10/1731735680811/Put/seqid=0 2024-11-16T05:41:22,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742264_1440 (size=31255) 2024-11-16T05:41:22,553 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/c2935ac895a440cc811fddab68f93dc5 2024-11-16T05:41:22,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/317c815719ec47d2a6ee82b312a93473 is 50, key is test_row_0/B:col10/1731735680811/Put/seqid=0 2024-11-16T05:41:22,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742265_1441 (size=12301) 2024-11-16T05:41:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:41:22,874 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/b55cae63336d4679ac3da41e8c21bf08 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/b55cae63336d4679ac3da41e8c21bf08 2024-11-16T05:41:22,878 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/A of 4d436cb8d7fad8c1b05b38c8375a7637 into b55cae63336d4679ac3da41e8c21bf08(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:22,878 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:22,878 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/A, priority=13, startTime=1731735682050; duration=0sec 2024-11-16T05:41:22,879 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:22,879 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:A 2024-11-16T05:41:22,979 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/317c815719ec47d2a6ee82b312a93473 2024-11-16T05:41:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:22,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/3d40db62165248659ef68c459b05cebf is 50, key is test_row_0/C:col10/1731735680811/Put/seqid=0 2024-11-16T05:41:23,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742266_1442 (size=12301) 2024-11-16T05:41:23,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735743092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:23,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735743094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:23,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735743198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:23,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735743208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:23,403 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/3d40db62165248659ef68c459b05cebf 2024-11-16T05:41:23,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/c2935ac895a440cc811fddab68f93dc5 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c2935ac895a440cc811fddab68f93dc5 2024-11-16T05:41:23,409 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c2935ac895a440cc811fddab68f93dc5, entries=150, sequenceid=288, filesize=30.5 K 2024-11-16T05:41:23,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/317c815719ec47d2a6ee82b312a93473 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/317c815719ec47d2a6ee82b312a93473 2024-11-16T05:41:23,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735743408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:23,417 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/317c815719ec47d2a6ee82b312a93473, entries=150, sequenceid=288, filesize=12.0 K 2024-11-16T05:41:23,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/3d40db62165248659ef68c459b05cebf as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3d40db62165248659ef68c459b05cebf 2024-11-16T05:41:23,421 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3d40db62165248659ef68c459b05cebf, entries=150, sequenceid=288, filesize=12.0 K 2024-11-16T05:41:23,422 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1294ms, sequenceid=288, compaction requested=false 2024-11-16T05:41:23,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:23,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:23,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-16T05:41:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-16T05:41:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:23,424 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:23,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-16T05:41:23,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-16T05:41:23,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7520 sec 2024-11-16T05:41:23,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.7570 sec 2024-11-16T05:41:23,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168503788f0b3d4af9ae3f06e197c0148f_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735683423/Put/seqid=0 2024-11-16T05:41:23,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742267_1443 (size=14994) 2024-11-16T05:41:23,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735743442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:23,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735743558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735743718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:23,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735743768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:41:23,818 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:23,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-16T05:41:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:41:23,820 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:23,821 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:23,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:23,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:23,842 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168503788f0b3d4af9ae3f06e197c0148f_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168503788f0b3d4af9ae3f06e197c0148f_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:23,843 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/4d8ba24b470c4730b826566f7b4e9271, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:23,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/4d8ba24b470c4730b826566f7b4e9271 is 175, key is test_row_0/A:col10/1731735683423/Put/seqid=0 2024-11-16T05:41:23,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742268_1444 (size=39949) 2024-11-16T05:41:23,899 INFO [Thread-1633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1633. 2024-11-16T05:41:23,899 DEBUG [Thread-1633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:23,899 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:23,900 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:23,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:23,903 INFO [Thread-1627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1627. 2024-11-16T05:41:23,903 DEBUG [Thread-1627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:23,903 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:23,903 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:23,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:23,904 INFO [Thread-1625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1625. 2024-11-16T05:41:23,904 DEBUG [Thread-1625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:23,904 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:23,904 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:23,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:23,906 INFO [Thread-1629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1629. 2024-11-16T05:41:23,906 DEBUG [Thread-1629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:23,906 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:23,906 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:23,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:23,909 INFO [Thread-1631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1631. 2024-11-16T05:41:23,909 DEBUG [Thread-1631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:23,909 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:23,910 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:23,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:41:23,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T05:41:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:23,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735744079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:24,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T05:41:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:41:24,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38504 deadline: 1731735744228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:24,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/4d8ba24b470c4730b826566f7b4e9271 2024-11-16T05:41:24,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/e9d4952570204bee8bcc781b2cd52c10 is 50, key is test_row_0/B:col10/1731735683423/Put/seqid=0 2024-11-16T05:41:24,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742269_1445 (size=12301) 2024-11-16T05:41:24,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T05:41:24,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:24,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T05:41:24,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:24,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:41:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38596 deadline: 1731735744588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:24,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T05:41:24,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:24,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/e9d4952570204bee8bcc781b2cd52c10 2024-11-16T05:41:24,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/cac3ecfd87e14cd3ac33b555f68c4120 is 50, key is test_row_0/C:col10/1731735683423/Put/seqid=0 2024-11-16T05:41:24,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742270_1446 (size=12301) 2024-11-16T05:41:24,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T05:41:24,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:24,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T05:41:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:24,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:41:25,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T05:41:25,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:25,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. as already flushing 2024-11-16T05:41:25,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:25,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:25,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:25,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/cac3ecfd87e14cd3ac33b555f68c4120 2024-11-16T05:41:25,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:25,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38546 deadline: 1731735745088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38582 deadline: 1731735745089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:25,090 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:25,090 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in 4d436cb8d7fad8c1b05b38c8375a7637 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:41:25,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/4d8ba24b470c4730b826566f7b4e9271 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4d8ba24b470c4730b826566f7b4e9271 2024-11-16T05:41:25,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4d8ba24b470c4730b826566f7b4e9271, entries=200, sequenceid=318, filesize=39.0 K 2024-11-16T05:41:25,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/e9d4952570204bee8bcc781b2cd52c10 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/e9d4952570204bee8bcc781b2cd52c10 2024-11-16T05:41:25,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/e9d4952570204bee8bcc781b2cd52c10, entries=150, sequenceid=318, filesize=12.0 K 2024-11-16T05:41:25,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/cac3ecfd87e14cd3ac33b555f68c4120 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cac3ecfd87e14cd3ac33b555f68c4120 2024-11-16T05:41:25,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cac3ecfd87e14cd3ac33b555f68c4120, entries=150, sequenceid=318, filesize=12.0 K 2024-11-16T05:41:25,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1690ms, sequenceid=318, compaction requested=true 2024-11-16T05:41:25,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:25,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:25,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:25,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:25,115 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:25,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:25,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d436cb8d7fad8c1b05b38c8375a7637:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:25,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:25,115 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:25,115 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:25,115 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:25,115 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/A is initiating minor compaction (all files) 2024-11-16T05:41:25,115 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/B is initiating minor compaction (all files) 2024-11-16T05:41:25,115 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/B in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:25,116 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/A in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:25,116 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/40a4924a0cfe4a5eb871615a7b84805e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/317c815719ec47d2a6ee82b312a93473, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/e9d4952570204bee8bcc781b2cd52c10] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=36.7 K 2024-11-16T05:41:25,116 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/b55cae63336d4679ac3da41e8c21bf08, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c2935ac895a440cc811fddab68f93dc5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4d8ba24b470c4730b826566f7b4e9271] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=100.7 K 2024-11-16T05:41:25,116 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:25,116 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/b55cae63336d4679ac3da41e8c21bf08, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c2935ac895a440cc811fddab68f93dc5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4d8ba24b470c4730b826566f7b4e9271] 2024-11-16T05:41:25,116 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a4924a0cfe4a5eb871615a7b84805e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731735678608 2024-11-16T05:41:25,116 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b55cae63336d4679ac3da41e8c21bf08, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731735678608 2024-11-16T05:41:25,116 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 317c815719ec47d2a6ee82b312a93473, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731735680806 2024-11-16T05:41:25,116 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2935ac895a440cc811fddab68f93dc5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731735680806 2024-11-16T05:41:25,116 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d4952570204bee8bcc781b2cd52c10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731735683031 2024-11-16T05:41:25,117 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8ba24b470c4730b826566f7b4e9271, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731735683031 2024-11-16T05:41:25,120 INFO [Thread-1614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1614. 2024-11-16T05:41:25,120 DEBUG [Thread-1614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:25,120 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:25,120 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:25,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:25,125 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#B#compaction#374 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:25,125 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/aea4f288cf0a4c3dbd4f8f26c8cef2f7 is 50, key is test_row_0/B:col10/1731735683423/Put/seqid=0 2024-11-16T05:41:25,126 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:25,128 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116554c3185281e4c7dbad5e443caba3333_4d436cb8d7fad8c1b05b38c8375a7637 store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:25,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742271_1447 (size=13051) 2024-11-16T05:41:25,157 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116554c3185281e4c7dbad5e443caba3333_4d436cb8d7fad8c1b05b38c8375a7637, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:25,157 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116554c3185281e4c7dbad5e443caba3333_4d436cb8d7fad8c1b05b38c8375a7637 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:25,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742272_1448 (size=4469) 2024-11-16T05:41:25,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T05:41:25,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:25,209 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T05:41:25,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169600112af5ed4fa185caec45e142cbfb_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735683428/Put/seqid=0 2024-11-16T05:41:25,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742273_1449 (size=12454) 2024-11-16T05:41:25,253 INFO [Thread-1618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1618. 2024-11-16T05:41:25,253 DEBUG [Thread-1618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:25,253 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:25,253 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:25,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:25,537 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/aea4f288cf0a4c3dbd4f8f26c8cef2f7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/aea4f288cf0a4c3dbd4f8f26c8cef2f7 2024-11-16T05:41:25,546 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/B of 4d436cb8d7fad8c1b05b38c8375a7637 into aea4f288cf0a4c3dbd4f8f26c8cef2f7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:25,546 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:25,546 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/B, priority=13, startTime=1731735685115; duration=0sec 2024-11-16T05:41:25,546 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:25,546 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:B 2024-11-16T05:41:25,547 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:25,548 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:25,548 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): 4d436cb8d7fad8c1b05b38c8375a7637/C is initiating minor compaction (all files) 2024-11-16T05:41:25,548 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d436cb8d7fad8c1b05b38c8375a7637/C in TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:25,549 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ef6cf7a6499443c0bc99737b84989f64, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3d40db62165248659ef68c459b05cebf, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cac3ecfd87e14cd3ac33b555f68c4120] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp, totalSize=36.7 K 2024-11-16T05:41:25,549 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ef6cf7a6499443c0bc99737b84989f64, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731735678608 2024-11-16T05:41:25,550 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d40db62165248659ef68c459b05cebf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731735680806 2024-11-16T05:41:25,550 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting cac3ecfd87e14cd3ac33b555f68c4120, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731735683031 2024-11-16T05:41:25,562 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#C#compaction#377 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:25,562 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/2d26db8ba0594aed8ca312d4af595992 is 50, key is test_row_0/C:col10/1731735683423/Put/seqid=0 2024-11-16T05:41:25,563 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d436cb8d7fad8c1b05b38c8375a7637#A#compaction#375 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:25,563 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/b5d796ce284c46c7835e25f8a0d75c33 is 175, key is test_row_0/A:col10/1731735683423/Put/seqid=0 2024-11-16T05:41:25,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742274_1450 (size=13051) 2024-11-16T05:41:25,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742275_1451 (size=32005) 2024-11-16T05:41:25,571 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/b5d796ce284c46c7835e25f8a0d75c33 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/b5d796ce284c46c7835e25f8a0d75c33 2024-11-16T05:41:25,574 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/A of 4d436cb8d7fad8c1b05b38c8375a7637 into b5d796ce284c46c7835e25f8a0d75c33(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:25,574 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:25,574 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/A, priority=13, startTime=1731735685114; duration=0sec 2024-11-16T05:41:25,575 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:25,575 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:A 2024-11-16T05:41:25,599 INFO [Thread-1616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1616. 2024-11-16T05:41:25,599 DEBUG [Thread-1616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:25,599 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:25,600 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:25,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:25,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:25,630 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169600112af5ed4fa185caec45e142cbfb_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169600112af5ed4fa185caec45e142cbfb_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:25,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/4f5fbf22118348e8a54a4ddb5452c29d, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:25,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/4f5fbf22118348e8a54a4ddb5452c29d is 175, key is test_row_0/A:col10/1731735683428/Put/seqid=0 2024-11-16T05:41:25,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742276_1452 (size=31255) 2024-11-16T05:41:25,973 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/2d26db8ba0594aed8ca312d4af595992 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/2d26db8ba0594aed8ca312d4af595992 2024-11-16T05:41:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:41:25,980 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d436cb8d7fad8c1b05b38c8375a7637/C of 4d436cb8d7fad8c1b05b38c8375a7637 into 2d26db8ba0594aed8ca312d4af595992(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:25,980 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:25,980 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., storeName=4d436cb8d7fad8c1b05b38c8375a7637/C, priority=13, startTime=1731735685115; duration=0sec 2024-11-16T05:41:25,980 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:25,980 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d436cb8d7fad8c1b05b38c8375a7637:C 2024-11-16T05:41:26,039 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/4f5fbf22118348e8a54a4ddb5452c29d 2024-11-16T05:41:26,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/b1afa0a9e8a64e248e64e084a76ad4c3 is 50, key is test_row_0/B:col10/1731735683428/Put/seqid=0 2024-11-16T05:41:26,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742277_1453 (size=12301) 2024-11-16T05:41:26,449 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/b1afa0a9e8a64e248e64e084a76ad4c3 2024-11-16T05:41:26,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/15ec8ee42bfe41eea6f72d364b9b949c is 50, key is test_row_0/C:col10/1731735683428/Put/seqid=0 2024-11-16T05:41:26,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742278_1454 (size=12301) 2024-11-16T05:41:26,867 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/15ec8ee42bfe41eea6f72d364b9b949c 2024-11-16T05:41:26,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/4f5fbf22118348e8a54a4ddb5452c29d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4f5fbf22118348e8a54a4ddb5452c29d 2024-11-16T05:41:26,881 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4f5fbf22118348e8a54a4ddb5452c29d, entries=150, sequenceid=326, filesize=30.5 K 2024-11-16T05:41:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/b1afa0a9e8a64e248e64e084a76ad4c3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/b1afa0a9e8a64e248e64e084a76ad4c3 2024-11-16T05:41:26,884 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/b1afa0a9e8a64e248e64e084a76ad4c3, entries=150, sequenceid=326, filesize=12.0 K 2024-11-16T05:41:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/15ec8ee42bfe41eea6f72d364b9b949c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/15ec8ee42bfe41eea6f72d364b9b949c 2024-11-16T05:41:26,887 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/15ec8ee42bfe41eea6f72d364b9b949c, entries=150, sequenceid=326, filesize=12.0 K 2024-11-16T05:41:26,888 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=13.42 KB/13740 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1679ms, sequenceid=326, compaction requested=false 2024-11-16T05:41:26,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 4d436cb8d7fad8c1b05b38c8375a7637: 2024-11-16T05:41:26,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:26,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-16T05:41:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-16T05:41:26,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-16T05:41:26,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0690 sec 2024-11-16T05:41:26,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 3.0730 sec 2024-11-16T05:41:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:41:27,990 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:33,555 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:41:35,173 INFO [Thread-1620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1620. 2024-11-16T05:41:35,173 DEBUG [Thread-1620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:35,173 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:35,173 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:35,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:35,181 INFO [Thread-1622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1622. 2024-11-16T05:41:35,181 DEBUG [Thread-1622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:35,181 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:35,181 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:35,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1583 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4749 rows 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1597 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4779 rows 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1595 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4776 rows 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1590 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4765 rows 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1579 2024-11-16T05:41:35,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4737 rows 2024-11-16T05:41:35,181 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:41:35,182 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:41:35,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:35,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:35,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:35,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:41:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:41:35,186 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735695186"}]},"ts":"1731735695186"} 2024-11-16T05:41:35,188 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:41:35,188 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:41:35,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:41:35,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, UNASSIGN}] 2024-11-16T05:41:35,191 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, UNASSIGN 2024-11-16T05:41:35,191 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=4d436cb8d7fad8c1b05b38c8375a7637, regionState=CLOSING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:35,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, UNASSIGN because future has completed 2024-11-16T05:41:35,193 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:41:35,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:41:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:41:35,347 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:35,347 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:41:35,348 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 4d436cb8d7fad8c1b05b38c8375a7637, disabling compactions & flushes 2024-11-16T05:41:35,348 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:35,348 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:35,348 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. after waiting 0 ms 2024-11-16T05:41:35,348 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:35,348 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 4d436cb8d7fad8c1b05b38c8375a7637 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-16T05:41:35,360 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f7fdaf38c9204955988c2c229da2332b_4d436cb8d7fad8c1b05b38c8375a7637 is 50, key is test_row_0/A:col10/1731735695179/Put/seqid=0 2024-11-16T05:41:35,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742279_1455 (size=9914) 2024-11-16T05:41:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:41:35,764 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:35,769 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f7fdaf38c9204955988c2c229da2332b_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f7fdaf38c9204955988c2c229da2332b_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:35,769 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/06d2ed2d1da143c09acfd653516b3ffe, store: [table=TestAcidGuarantees family=A region=4d436cb8d7fad8c1b05b38c8375a7637] 2024-11-16T05:41:35,770 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/06d2ed2d1da143c09acfd653516b3ffe is 175, key is test_row_0/A:col10/1731735695179/Put/seqid=0 2024-11-16T05:41:35,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742280_1456 (size=22561) 2024-11-16T05:41:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:41:36,174 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/06d2ed2d1da143c09acfd653516b3ffe 2024-11-16T05:41:36,187 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/660e407fcfee4646af8d1a8c4420ee96 is 50, key is test_row_0/B:col10/1731735695179/Put/seqid=0 2024-11-16T05:41:36,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742281_1457 (size=9857) 2024-11-16T05:41:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:41:36,592 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/660e407fcfee4646af8d1a8c4420ee96 2024-11-16T05:41:36,605 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/ac9aac8ffbdf45549a381a7b7ae9fad8 is 50, key is test_row_0/C:col10/1731735695179/Put/seqid=0 2024-11-16T05:41:36,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742282_1458 (size=9857) 2024-11-16T05:41:37,010 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/ac9aac8ffbdf45549a381a7b7ae9fad8 2024-11-16T05:41:37,020 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/A/06d2ed2d1da143c09acfd653516b3ffe as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/06d2ed2d1da143c09acfd653516b3ffe 2024-11-16T05:41:37,024 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/06d2ed2d1da143c09acfd653516b3ffe, entries=100, sequenceid=336, filesize=22.0 K 2024-11-16T05:41:37,024 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/B/660e407fcfee4646af8d1a8c4420ee96 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/660e407fcfee4646af8d1a8c4420ee96 2024-11-16T05:41:37,027 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/660e407fcfee4646af8d1a8c4420ee96, entries=100, sequenceid=336, filesize=9.6 K 2024-11-16T05:41:37,027 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/.tmp/C/ac9aac8ffbdf45549a381a7b7ae9fad8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ac9aac8ffbdf45549a381a7b7ae9fad8 2024-11-16T05:41:37,030 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ac9aac8ffbdf45549a381a7b7ae9fad8, entries=100, sequenceid=336, filesize=9.6 K 2024-11-16T05:41:37,031 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1683ms, sequenceid=336, compaction requested=true 2024-11-16T05:41:37,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/8ac3812fc0b94290be5f93838a978b91, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/721bdd7c73084b4884cdb7e49d05652f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0e7de23e5bc8405cb3d168528760940c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/9dc2b601f478439da9f93463cb81727f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/04beb7d19e214a97b20239052f72369d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/dc735fa64b104f8c9395bd2d251e6b6e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/5117aecbfcbd4ebea9f0af59e42b67d5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/297f5fb48e69477bb53ca6fb64240beb, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/59b7bface58845f9b887b5643f72e643, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2c84ff7ec5984e98ace38cb11afeb315, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/1d5588456cf74707a140ac56f7ca9c81, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/e803cf73e4cf4817a54121099b1f48cf, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2acda43d2eed4e858eb2808f3b9d9854, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ea3800a39948488cbe72f33a9e52dff7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2b9d11e4823d422bb56bf0dd33abcb54, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/731e58ab729f4f2c98b0c373644fc933, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c63628a00c6346cfacd8923bfa39797b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ada14fca885f4ca7998225f64586db5a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/b55cae63336d4679ac3da41e8c21bf08, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c2935ac895a440cc811fddab68f93dc5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4d8ba24b470c4730b826566f7b4e9271] to archive 2024-11-16T05:41:37,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:41:37,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/8ac3812fc0b94290be5f93838a978b91 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/8ac3812fc0b94290be5f93838a978b91 2024-11-16T05:41:37,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/721bdd7c73084b4884cdb7e49d05652f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/721bdd7c73084b4884cdb7e49d05652f 2024-11-16T05:41:37,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0e7de23e5bc8405cb3d168528760940c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0e7de23e5bc8405cb3d168528760940c 2024-11-16T05:41:37,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/9dc2b601f478439da9f93463cb81727f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/9dc2b601f478439da9f93463cb81727f 2024-11-16T05:41:37,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/0be07d3c8eb947c9bb49dc7cd6a6e5bf 2024-11-16T05:41:37,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/04beb7d19e214a97b20239052f72369d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/04beb7d19e214a97b20239052f72369d 2024-11-16T05:41:37,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/dc735fa64b104f8c9395bd2d251e6b6e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/dc735fa64b104f8c9395bd2d251e6b6e 2024-11-16T05:41:37,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/5117aecbfcbd4ebea9f0af59e42b67d5 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/5117aecbfcbd4ebea9f0af59e42b67d5 2024-11-16T05:41:37,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/297f5fb48e69477bb53ca6fb64240beb to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/297f5fb48e69477bb53ca6fb64240beb 2024-11-16T05:41:37,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/59b7bface58845f9b887b5643f72e643 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/59b7bface58845f9b887b5643f72e643 2024-11-16T05:41:37,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2c84ff7ec5984e98ace38cb11afeb315 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2c84ff7ec5984e98ace38cb11afeb315 2024-11-16T05:41:37,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/1d5588456cf74707a140ac56f7ca9c81 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/1d5588456cf74707a140ac56f7ca9c81 2024-11-16T05:41:37,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/e803cf73e4cf4817a54121099b1f48cf to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/e803cf73e4cf4817a54121099b1f48cf 2024-11-16T05:41:37,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2acda43d2eed4e858eb2808f3b9d9854 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2acda43d2eed4e858eb2808f3b9d9854 2024-11-16T05:41:37,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ea3800a39948488cbe72f33a9e52dff7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ea3800a39948488cbe72f33a9e52dff7 2024-11-16T05:41:37,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2b9d11e4823d422bb56bf0dd33abcb54 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/2b9d11e4823d422bb56bf0dd33abcb54 2024-11-16T05:41:37,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/731e58ab729f4f2c98b0c373644fc933 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/731e58ab729f4f2c98b0c373644fc933 2024-11-16T05:41:37,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c63628a00c6346cfacd8923bfa39797b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c63628a00c6346cfacd8923bfa39797b 2024-11-16T05:41:37,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ada14fca885f4ca7998225f64586db5a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/ada14fca885f4ca7998225f64586db5a 2024-11-16T05:41:37,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/b55cae63336d4679ac3da41e8c21bf08 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/b55cae63336d4679ac3da41e8c21bf08 2024-11-16T05:41:37,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c2935ac895a440cc811fddab68f93dc5 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/c2935ac895a440cc811fddab68f93dc5 2024-11-16T05:41:37,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4d8ba24b470c4730b826566f7b4e9271 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4d8ba24b470c4730b826566f7b4e9271 2024-11-16T05:41:37,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/269daa25fd2e4526aae338deb366125a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/94fca42f257a412b9f7ebed3d0dbfb2b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/3096e165422647d5a14424badf70bf8f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d63656706b1b41c0a61bea7438aa26a7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/8edc2326bb864097a9c39497284a4e52, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/b9f7061c1c204638ab01cb407d2e1429, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5c89b5370a4040038f38d8b941461586, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/2129f41e179848f6aa09908dde3e4442, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/ed89415f85214a29b0598fb03bcbb047, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8436a2e52974e97a4069efe84f0989a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c62e0b5f535e4aa7a2b13b44962ad750, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/1152d97b11f24f2ca6a91cce51c233f9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8e4580293334a009586aa471401f221, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/6667dabb53574eceb41bc37ba8a4e444, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c92d7dfbbddc4ba3a1042d975629a981, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/3636d232af1343d9bebff6ac78c61d16, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5b37de0fc0a144f0ac7200dda3abf2f6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/7ac0e380cf7f43cc8011c3a1888ca164, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/40a4924a0cfe4a5eb871615a7b84805e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/27b8ce1e432a4ab2bafe3beaf961b325, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/317c815719ec47d2a6ee82b312a93473, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/e9d4952570204bee8bcc781b2cd52c10] to archive 2024-11-16T05:41:37,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:41:37,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/269daa25fd2e4526aae338deb366125a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/269daa25fd2e4526aae338deb366125a 2024-11-16T05:41:37,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/94fca42f257a412b9f7ebed3d0dbfb2b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/94fca42f257a412b9f7ebed3d0dbfb2b 2024-11-16T05:41:37,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/3096e165422647d5a14424badf70bf8f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/3096e165422647d5a14424badf70bf8f 2024-11-16T05:41:37,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d63656706b1b41c0a61bea7438aa26a7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d63656706b1b41c0a61bea7438aa26a7 2024-11-16T05:41:37,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/8edc2326bb864097a9c39497284a4e52 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/8edc2326bb864097a9c39497284a4e52 2024-11-16T05:41:37,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/b9f7061c1c204638ab01cb407d2e1429 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/b9f7061c1c204638ab01cb407d2e1429 2024-11-16T05:41:37,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5c89b5370a4040038f38d8b941461586 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5c89b5370a4040038f38d8b941461586 2024-11-16T05:41:37,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/2129f41e179848f6aa09908dde3e4442 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/2129f41e179848f6aa09908dde3e4442 2024-11-16T05:41:37,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/ed89415f85214a29b0598fb03bcbb047 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/ed89415f85214a29b0598fb03bcbb047 2024-11-16T05:41:37,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8436a2e52974e97a4069efe84f0989a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8436a2e52974e97a4069efe84f0989a 2024-11-16T05:41:37,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c62e0b5f535e4aa7a2b13b44962ad750 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c62e0b5f535e4aa7a2b13b44962ad750 2024-11-16T05:41:37,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/1152d97b11f24f2ca6a91cce51c233f9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/1152d97b11f24f2ca6a91cce51c233f9 2024-11-16T05:41:37,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8e4580293334a009586aa471401f221 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/d8e4580293334a009586aa471401f221 2024-11-16T05:41:37,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/6667dabb53574eceb41bc37ba8a4e444 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/6667dabb53574eceb41bc37ba8a4e444 2024-11-16T05:41:37,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c92d7dfbbddc4ba3a1042d975629a981 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/c92d7dfbbddc4ba3a1042d975629a981 2024-11-16T05:41:37,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/3636d232af1343d9bebff6ac78c61d16 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/3636d232af1343d9bebff6ac78c61d16 2024-11-16T05:41:37,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5b37de0fc0a144f0ac7200dda3abf2f6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/5b37de0fc0a144f0ac7200dda3abf2f6 2024-11-16T05:41:37,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/7ac0e380cf7f43cc8011c3a1888ca164 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/7ac0e380cf7f43cc8011c3a1888ca164 2024-11-16T05:41:37,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/40a4924a0cfe4a5eb871615a7b84805e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/40a4924a0cfe4a5eb871615a7b84805e 2024-11-16T05:41:37,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/27b8ce1e432a4ab2bafe3beaf961b325 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/27b8ce1e432a4ab2bafe3beaf961b325 2024-11-16T05:41:37,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/317c815719ec47d2a6ee82b312a93473 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/317c815719ec47d2a6ee82b312a93473 2024-11-16T05:41:37,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/e9d4952570204bee8bcc781b2cd52c10 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/e9d4952570204bee8bcc781b2cd52c10 2024-11-16T05:41:37,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/4a27e63849be44bbbb03f2783f77c51d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c29c93541c4a47bbb71f4eda94b42225, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cecb92ac5cc6462c930a466981c878b5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c7b674c0c62e4d01b32627a66459d2fd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/5f84df10ac664eba8db54c7f8c51a9c1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ae23e5ff211f42a3bfbdbcc72acce438, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/582c7f4e5d02423ea9344282dadfcb15, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/e44e0b738a674893b92e00fe7430671a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/29ce4d6506394d27bd85007efbf84d09, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/49ab696583374845a1bb980170c633aa, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/986509a4f50c4515898d4314f24d62e1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/1466e08e589144eea03b79a58d22c1f3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cface2354c6d4173adf75b26a0dd8188, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/b5c9ac1bbf9a4ccb92bcd2e05955676a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/f4bd030552fd43b2a42903fb9034f8fa, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3cb81c3a6d824b4e8bd1cf6f6ca6962d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/440fbcadc13c4e409856dcfe0f29a425, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/a9109d77d8ca42f6a982e653ed4a6fa8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ef6cf7a6499443c0bc99737b84989f64, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ba12af656c7840dc860de76f3dba114d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3d40db62165248659ef68c459b05cebf, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cac3ecfd87e14cd3ac33b555f68c4120] to archive 2024-11-16T05:41:37,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:41:37,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/4a27e63849be44bbbb03f2783f77c51d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/4a27e63849be44bbbb03f2783f77c51d 2024-11-16T05:41:37,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c29c93541c4a47bbb71f4eda94b42225 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c29c93541c4a47bbb71f4eda94b42225 2024-11-16T05:41:37,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cecb92ac5cc6462c930a466981c878b5 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cecb92ac5cc6462c930a466981c878b5 2024-11-16T05:41:37,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c7b674c0c62e4d01b32627a66459d2fd to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/c7b674c0c62e4d01b32627a66459d2fd 2024-11-16T05:41:37,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/5f84df10ac664eba8db54c7f8c51a9c1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/5f84df10ac664eba8db54c7f8c51a9c1 2024-11-16T05:41:37,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ae23e5ff211f42a3bfbdbcc72acce438 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ae23e5ff211f42a3bfbdbcc72acce438 2024-11-16T05:41:37,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/582c7f4e5d02423ea9344282dadfcb15 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/582c7f4e5d02423ea9344282dadfcb15 2024-11-16T05:41:37,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/e44e0b738a674893b92e00fe7430671a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/e44e0b738a674893b92e00fe7430671a 2024-11-16T05:41:37,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/29ce4d6506394d27bd85007efbf84d09 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/29ce4d6506394d27bd85007efbf84d09 2024-11-16T05:41:37,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/49ab696583374845a1bb980170c633aa to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/49ab696583374845a1bb980170c633aa 2024-11-16T05:41:37,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/986509a4f50c4515898d4314f24d62e1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/986509a4f50c4515898d4314f24d62e1 2024-11-16T05:41:37,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/1466e08e589144eea03b79a58d22c1f3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/1466e08e589144eea03b79a58d22c1f3 2024-11-16T05:41:37,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cface2354c6d4173adf75b26a0dd8188 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cface2354c6d4173adf75b26a0dd8188 2024-11-16T05:41:37,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/b5c9ac1bbf9a4ccb92bcd2e05955676a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/b5c9ac1bbf9a4ccb92bcd2e05955676a 2024-11-16T05:41:37,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/f4bd030552fd43b2a42903fb9034f8fa to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/f4bd030552fd43b2a42903fb9034f8fa 2024-11-16T05:41:37,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3cb81c3a6d824b4e8bd1cf6f6ca6962d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3cb81c3a6d824b4e8bd1cf6f6ca6962d 2024-11-16T05:41:37,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/440fbcadc13c4e409856dcfe0f29a425 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/440fbcadc13c4e409856dcfe0f29a425 2024-11-16T05:41:37,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/a9109d77d8ca42f6a982e653ed4a6fa8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/a9109d77d8ca42f6a982e653ed4a6fa8 2024-11-16T05:41:37,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ef6cf7a6499443c0bc99737b84989f64 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ef6cf7a6499443c0bc99737b84989f64 2024-11-16T05:41:37,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ba12af656c7840dc860de76f3dba114d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ba12af656c7840dc860de76f3dba114d 2024-11-16T05:41:37,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3d40db62165248659ef68c459b05cebf to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/3d40db62165248659ef68c459b05cebf 2024-11-16T05:41:37,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cac3ecfd87e14cd3ac33b555f68c4120 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/cac3ecfd87e14cd3ac33b555f68c4120 2024-11-16T05:41:37,089 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/recovered.edits/339.seqid, newMaxSeqId=339, maxSeqId=4 2024-11-16T05:41:37,089 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. 2024-11-16T05:41:37,089 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 4d436cb8d7fad8c1b05b38c8375a7637: Waiting for close lock at 1731735695348Running coprocessor pre-close hooks at 1731735695348Disabling compacts and flushes for region at 1731735695348Disabling writes for close at 1731735695348Obtaining lock to block concurrent updates at 1731735695348Preparing flush snapshotting stores in 4d436cb8d7fad8c1b05b38c8375a7637 at 1731735695348Finished memstore snapshotting TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731735695349 (+1 ms)Flushing stores of TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637. at 1731735695350 (+1 ms)Flushing 4d436cb8d7fad8c1b05b38c8375a7637/A: creating writer at 1731735695350Flushing mob file 4d436cb8d7fad8c1b05b38c8375a7637/A: appending metadata at 1731735695360 (+10 ms)Flushing mob file 4d436cb8d7fad8c1b05b38c8375a7637/A: closing flushed file at 1731735695360Flushing 4d436cb8d7fad8c1b05b38c8375a7637/A: appending metadata at 1731735695770 (+410 ms)Flushing 4d436cb8d7fad8c1b05b38c8375a7637/A: closing flushed file at 1731735695770Flushing 4d436cb8d7fad8c1b05b38c8375a7637/B: creating writer at 1731735696182 (+412 ms)Flushing 4d436cb8d7fad8c1b05b38c8375a7637/B: appending metadata at 1731735696187 (+5 ms)Flushing 4d436cb8d7fad8c1b05b38c8375a7637/B: closing flushed file at 1731735696187Flushing 4d436cb8d7fad8c1b05b38c8375a7637/C: creating writer at 1731735696600 (+413 ms)Flushing 4d436cb8d7fad8c1b05b38c8375a7637/C: appending metadata at 1731735696604 (+4 ms)Flushing 4d436cb8d7fad8c1b05b38c8375a7637/C: closing flushed file at 1731735696604Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32412a4a: reopening flushed file at 1731735697019 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2a237a: reopening flushed file at 1731735697024 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bcc9d08: reopening flushed file at 1731735697027 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 4d436cb8d7fad8c1b05b38c8375a7637 in 1683ms, sequenceid=336, compaction requested=true at 1731735697031 (+4 ms)Writing region close event to WAL at 1731735697086 (+55 ms)Running coprocessor post-close hooks at 1731735697089 (+3 ms)Closed at 1731735697089 2024-11-16T05:41:37,091 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,091 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=4d436cb8d7fad8c1b05b38c8375a7637, regionState=CLOSED 2024-11-16T05:41:37,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:41:37,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-16T05:41:37,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 4d436cb8d7fad8c1b05b38c8375a7637, server=3456ee6a3164,35555,1731735575219 in 1.9000 sec 2024-11-16T05:41:37,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-16T05:41:37,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4d436cb8d7fad8c1b05b38c8375a7637, UNASSIGN in 1.9050 sec 2024-11-16T05:41:37,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-16T05:41:37,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9080 sec 2024-11-16T05:41:37,099 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735697099"}]},"ts":"1731735697099"} 2024-11-16T05:41:37,100 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:41:37,100 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:41:37,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9180 sec 2024-11-16T05:41:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:41:37,339 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:37,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:41:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:37,344 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T05:41:37,345 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:37,349 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,354 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/recovered.edits] 2024-11-16T05:41:37,361 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/06d2ed2d1da143c09acfd653516b3ffe to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/06d2ed2d1da143c09acfd653516b3ffe 2024-11-16T05:41:37,363 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4f5fbf22118348e8a54a4ddb5452c29d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/4f5fbf22118348e8a54a4ddb5452c29d 2024-11-16T05:41:37,365 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/b5d796ce284c46c7835e25f8a0d75c33 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/A/b5d796ce284c46c7835e25f8a0d75c33 2024-11-16T05:41:37,368 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/660e407fcfee4646af8d1a8c4420ee96 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/660e407fcfee4646af8d1a8c4420ee96 2024-11-16T05:41:37,370 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/aea4f288cf0a4c3dbd4f8f26c8cef2f7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/aea4f288cf0a4c3dbd4f8f26c8cef2f7 2024-11-16T05:41:37,372 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/b1afa0a9e8a64e248e64e084a76ad4c3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/B/b1afa0a9e8a64e248e64e084a76ad4c3 2024-11-16T05:41:37,376 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/15ec8ee42bfe41eea6f72d364b9b949c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/15ec8ee42bfe41eea6f72d364b9b949c 2024-11-16T05:41:37,377 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/2d26db8ba0594aed8ca312d4af595992 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/2d26db8ba0594aed8ca312d4af595992 2024-11-16T05:41:37,379 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ac9aac8ffbdf45549a381a7b7ae9fad8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/C/ac9aac8ffbdf45549a381a7b7ae9fad8 2024-11-16T05:41:37,383 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/recovered.edits/339.seqid to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637/recovered.edits/339.seqid 2024-11-16T05:41:37,384 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,384 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:41:37,385 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:41:37,386 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T05:41:37,390 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111601b7f3c9c5404575b29e71b6c9601ee5_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111601b7f3c9c5404575b29e71b6c9601ee5_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,392 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161517bdcb68224ad8b0bc91cc147edc96_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161517bdcb68224ad8b0bc91cc147edc96_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,393 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111617e964fd748c4fd1aa7002512575ac67_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111617e964fd748c4fd1aa7002512575ac67_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,395 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162a16ce837d824683bfc8be8b1dc75bf8_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162a16ce837d824683bfc8be8b1dc75bf8_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,396 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162ab521019dc3415197026821e5641981_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162ab521019dc3415197026821e5641981_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,397 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111652f7223b0a2b404c8362582be1eb7313_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111652f7223b0a2b404c8362582be1eb7313_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,398 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166176a4ec3af44f2a99517cbe7d899b2f_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166176a4ec3af44f2a99517cbe7d899b2f_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,399 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167c6b56c3c3374acb9b55ca348a89363a_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167c6b56c3c3374acb9b55ca348a89363a_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,399 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168503788f0b3d4af9ae3f06e197c0148f_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168503788f0b3d4af9ae3f06e197c0148f_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,400 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111686fbe1289a1f46cebe56f6c759dd88d5_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111686fbe1289a1f46cebe56f6c759dd88d5_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,401 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169600112af5ed4fa185caec45e142cbfb_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169600112af5ed4fa185caec45e142cbfb_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,401 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b3cd0ef7f8db42ca9d824c7e0d2448aa_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b3cd0ef7f8db42ca9d824c7e0d2448aa_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,402 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da478181c5064afa93071564e2c7bbbf_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da478181c5064afa93071564e2c7bbbf_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,403 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dfda4de6dfb843bb9ffc39e0d7417108_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dfda4de6dfb843bb9ffc39e0d7417108_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,403 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ec8fa1d899a445afb3fc53d18e59d22e_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ec8fa1d899a445afb3fc53d18e59d22e_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,404 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f7fdaf38c9204955988c2c229da2332b_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f7fdaf38c9204955988c2c229da2332b_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,405 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fb0b83cc5a5744c4a6ece653dff20eb0_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fb0b83cc5a5744c4a6ece653dff20eb0_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,405 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fdde4f6fefc0493abcdc72811f5f41f6_4d436cb8d7fad8c1b05b38c8375a7637 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fdde4f6fefc0493abcdc72811f5f41f6_4d436cb8d7fad8c1b05b38c8375a7637 2024-11-16T05:41:37,406 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:41:37,407 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:37,410 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:41:37,412 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:41:37,413 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:37,413 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:41:37,413 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731735697413"}]},"ts":"9223372036854775807"} 2024-11-16T05:41:37,415 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:41:37,415 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4d436cb8d7fad8c1b05b38c8375a7637, NAME => 'TestAcidGuarantees,,1731735661771.4d436cb8d7fad8c1b05b38c8375a7637.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:41:37,415 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:41:37,416 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731735697415"}]},"ts":"9223372036854775807"} 2024-11-16T05:41:37,417 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:41:37,418 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:37,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-16T05:41:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T05:41:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:41:37,452 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:37,462 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 241), OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 481), ProcessCount=11 (was 11), AvailableMemoryMB=3365 (was 3609) 2024-11-16T05:41:37,472 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=3365 2024-11-16T05:41:37,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:41:37,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:41:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:41:37,476 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:41:37,476 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:37,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-16T05:41:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T05:41:37,477 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:41:37,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742283_1459 (size=959) 2024-11-16T05:41:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T05:41:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T05:41:37,890 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d 2024-11-16T05:41:37,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742284_1460 (size=53) 2024-11-16T05:41:37,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:41:37,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing df3e8cf5ba630b1a9dfd75e69902d58f, disabling compactions & flushes 2024-11-16T05:41:37,897 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:37,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:37,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. after waiting 0 ms 2024-11-16T05:41:37,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:37,897 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:37,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for df3e8cf5ba630b1a9dfd75e69902d58f: Waiting for close lock at 1731735697897Disabling compacts and flushes for region at 1731735697897Disabling writes for close at 1731735697897Writing region close event to WAL at 1731735697897Closed at 1731735697897 2024-11-16T05:41:37,898 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:41:37,898 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731735697898"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731735697898"}]},"ts":"1731735697898"} 2024-11-16T05:41:37,900 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:41:37,900 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:41:37,900 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735697900"}]},"ts":"1731735697900"} 2024-11-16T05:41:37,901 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:41:37,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df3e8cf5ba630b1a9dfd75e69902d58f, ASSIGN}] 2024-11-16T05:41:37,903 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df3e8cf5ba630b1a9dfd75e69902d58f, ASSIGN 2024-11-16T05:41:37,903 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df3e8cf5ba630b1a9dfd75e69902d58f, ASSIGN; state=OFFLINE, location=3456ee6a3164,35555,1731735575219; forceNewPlan=false, retain=false 2024-11-16T05:41:38,054 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=df3e8cf5ba630b1a9dfd75e69902d58f, regionState=OPENING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df3e8cf5ba630b1a9dfd75e69902d58f, ASSIGN because future has completed 2024-11-16T05:41:38,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:41:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T05:41:38,211 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:38,211 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:41:38,212 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,212 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:41:38,212 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,212 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,213 INFO [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,215 INFO [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df3e8cf5ba630b1a9dfd75e69902d58f columnFamilyName A 2024-11-16T05:41:38,215 DEBUG [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:38,215 INFO [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] regionserver.HStore(327): Store=df3e8cf5ba630b1a9dfd75e69902d58f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:41:38,215 INFO [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,217 INFO [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df3e8cf5ba630b1a9dfd75e69902d58f columnFamilyName B 2024-11-16T05:41:38,217 DEBUG [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:38,217 INFO [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] regionserver.HStore(327): Store=df3e8cf5ba630b1a9dfd75e69902d58f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:41:38,217 INFO [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,218 INFO [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df3e8cf5ba630b1a9dfd75e69902d58f columnFamilyName C 2024-11-16T05:41:38,218 DEBUG [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:41:38,219 INFO [StoreOpener-df3e8cf5ba630b1a9dfd75e69902d58f-1 {}] regionserver.HStore(327): Store=df3e8cf5ba630b1a9dfd75e69902d58f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:41:38,219 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,220 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,220 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,221 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,221 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,221 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:41:38,222 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,223 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:41:38,224 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened df3e8cf5ba630b1a9dfd75e69902d58f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67221962, jitterRate=0.0016852915287017822}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:41:38,224 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,224 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for df3e8cf5ba630b1a9dfd75e69902d58f: Running coprocessor pre-open hook at 1731735698212Writing region info on filesystem at 1731735698212Initializing all the Stores at 1731735698213 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735698213Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735698213Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735698213Cleaning up temporary data from old regions at 1731735698221 (+8 ms)Running coprocessor post-open hooks at 1731735698224 (+3 ms)Region opened successfully at 1731735698224 2024-11-16T05:41:38,225 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., pid=126, masterSystemTime=1731735698208 2024-11-16T05:41:38,227 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:38,227 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:38,227 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=df3e8cf5ba630b1a9dfd75e69902d58f, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:41:38,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-16T05:41:38,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 in 173 msec 2024-11-16T05:41:38,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-16T05:41:38,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df3e8cf5ba630b1a9dfd75e69902d58f, ASSIGN in 328 msec 2024-11-16T05:41:38,232 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:41:38,233 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735698232"}]},"ts":"1731735698232"} 2024-11-16T05:41:38,234 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:41:38,235 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:41:38,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 761 msec 2024-11-16T05:41:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T05:41:38,619 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:38,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3005a7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,625 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b25fff6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,630 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61771250, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,632 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,635 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:41:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,39409,1731735574536 2024-11-16T05:41:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ca80cf8 2024-11-16T05:41:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:41:38,637 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:41:38,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,640 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c16ee09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,642 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55abb9d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2349d4d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,646 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@496ff7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,648 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32cea077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,650 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ab931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,652 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587193b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,654 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600df9c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,656 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d3b416d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,658 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@edf37ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,660 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c3e76e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,662 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab3596b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,665 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e0c16e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,666 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a58492b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a939df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65469ceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,671 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@310bbd75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,673 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acc58fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:41:38,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:41:38,675 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:41:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:41:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:41:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c8bcde9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:41:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:41:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:38,676 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:41:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d30947, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:41:38,677 DEBUG [Thread-2063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,677 DEBUG [Thread-2050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,677 DEBUG [Thread-2054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,677 DEBUG [Thread-2067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,677 DEBUG [Thread-2052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,678 DEBUG [Thread-2061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,679 DEBUG [Thread-2058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,679 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,680 DEBUG [Thread-2069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-16T05:41:38,681 DEBUG [Thread-2056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:41:38,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:41:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,682 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:41:38,682 DEBUG [Thread-2065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:41:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,683 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:38,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:41:38,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:41:38,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:41:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:41:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:41:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:41:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:41:38,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:41:38,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:41:38,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:41:38,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2] 2024-11-16T05:41:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:38,693 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:38,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T05:41:38,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/0b6ace7d1ee240efb05cfa1442dffc54 is 50, key is test_row_0/A:col10/1731735698693/Put/seqid=0 2024-11-16T05:41:38,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742285_1461 (size=12001) 2024-11-16T05:41:38,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735758731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735758732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:38,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735758733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:38,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735758733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735758734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:41:38,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:41:38,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:38,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:38,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:38,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:38,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:38,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735758848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735758848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735758848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735758848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735758848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:38,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:41:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:38,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:41:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735759058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735759058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735759058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735759058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735759058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/0b6ace7d1ee240efb05cfa1442dffc54 2024-11-16T05:41:39,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:41:39,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/85782b287c454bd7a181c9a0a730b827 is 50, key is test_row_0/B:col10/1731735698693/Put/seqid=0 2024-11-16T05:41:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742286_1462 (size=12001) 2024-11-16T05:41:39,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:41:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:41:39,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735759368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735759368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735759368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735759368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735759369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:41:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/85782b287c454bd7a181c9a0a730b827 2024-11-16T05:41:39,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/d15f92ee08f5452ab1dc2a0b51aa05df is 50, key is test_row_0/C:col10/1731735698693/Put/seqid=0 2024-11-16T05:41:39,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742287_1463 (size=12001) 2024-11-16T05:41:39,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:41:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:41:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:41:39,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735759878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735759878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735759888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735759888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735759888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:39,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:41:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:39,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:39,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/d15f92ee08f5452ab1dc2a0b51aa05df 2024-11-16T05:41:39,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/0b6ace7d1ee240efb05cfa1442dffc54 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/0b6ace7d1ee240efb05cfa1442dffc54 2024-11-16T05:41:39,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/0b6ace7d1ee240efb05cfa1442dffc54, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T05:41:39,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/85782b287c454bd7a181c9a0a730b827 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/85782b287c454bd7a181c9a0a730b827 2024-11-16T05:41:39,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/85782b287c454bd7a181c9a0a730b827, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T05:41:39,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/d15f92ee08f5452ab1dc2a0b51aa05df as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d15f92ee08f5452ab1dc2a0b51aa05df 2024-11-16T05:41:39,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d15f92ee08f5452ab1dc2a0b51aa05df, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T05:41:39,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for df3e8cf5ba630b1a9dfd75e69902d58f in 1299ms, sequenceid=14, compaction requested=false 2024-11-16T05:41:39,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:40,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:41:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:40,059 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T05:41:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/9c5b9774bda44120b12a0f4437784bed is 50, key is test_row_0/A:col10/1731735698733/Put/seqid=0 2024-11-16T05:41:40,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742288_1464 (size=12001) 2024-11-16T05:41:40,471 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/9c5b9774bda44120b12a0f4437784bed 2024-11-16T05:41:40,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/e3fc23a83b5a405c9e73036d7d93afe7 is 50, key is test_row_0/B:col10/1731735698733/Put/seqid=0 2024-11-16T05:41:40,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742289_1465 (size=12001) 2024-11-16T05:41:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:41:40,884 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/e3fc23a83b5a405c9e73036d7d93afe7 2024-11-16T05:41:40,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/7c124dee3987478f8abdc7aa0c34ae6e is 50, key is test_row_0/C:col10/1731735698733/Put/seqid=0 2024-11-16T05:41:40,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742290_1466 (size=12001) 2024-11-16T05:41:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:40,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:40,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735760911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:40,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:40,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:40,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735760912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735760913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735760914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735760914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735761028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735761028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735761028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735761028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735761029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735761238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735761238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735761238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735761239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735761239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,295 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/7c124dee3987478f8abdc7aa0c34ae6e 2024-11-16T05:41:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/9c5b9774bda44120b12a0f4437784bed as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9c5b9774bda44120b12a0f4437784bed 2024-11-16T05:41:41,303 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9c5b9774bda44120b12a0f4437784bed, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T05:41:41,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/e3fc23a83b5a405c9e73036d7d93afe7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/e3fc23a83b5a405c9e73036d7d93afe7 2024-11-16T05:41:41,307 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/e3fc23a83b5a405c9e73036d7d93afe7, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T05:41:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/7c124dee3987478f8abdc7aa0c34ae6e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/7c124dee3987478f8abdc7aa0c34ae6e 2024-11-16T05:41:41,311 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/7c124dee3987478f8abdc7aa0c34ae6e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T05:41:41,312 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for df3e8cf5ba630b1a9dfd75e69902d58f in 1253ms, sequenceid=37, compaction requested=false 2024-11-16T05:41:41,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:41,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:41,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-16T05:41:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-16T05:41:41,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-16T05:41:41,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6300 sec 2024-11-16T05:41:41,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.6370 sec 2024-11-16T05:41:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:41,552 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:41,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T05:41:41,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/5eb4ba1f14f94bbeb63e16d4172d05dd is 50, key is test_row_0/A:col10/1731735701552/Put/seqid=0 2024-11-16T05:41:41,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742291_1467 (size=16681) 2024-11-16T05:41:41,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/5eb4ba1f14f94bbeb63e16d4172d05dd 2024-11-16T05:41:41,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/8e6cf6af52cc491487d654b686d0d6b1 is 50, key is test_row_0/B:col10/1731735701552/Put/seqid=0 2024-11-16T05:41:41,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742292_1468 (size=12001) 2024-11-16T05:41:41,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735761585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735761588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735761589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735761590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735761592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735761698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735761698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735761698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735761708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735761708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735761908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735761908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735761908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735761918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735761918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:41,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:41,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/8e6cf6af52cc491487d654b686d0d6b1 2024-11-16T05:41:41,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/3ce17a6f16244293949aba393bfc2178 is 50, key is test_row_0/C:col10/1731735701552/Put/seqid=0 2024-11-16T05:41:41,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742293_1469 (size=12001) 2024-11-16T05:41:42,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735762217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735762218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735762228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735762228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735762238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/3ce17a6f16244293949aba393bfc2178 2024-11-16T05:41:42,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/5eb4ba1f14f94bbeb63e16d4172d05dd as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/5eb4ba1f14f94bbeb63e16d4172d05dd 2024-11-16T05:41:42,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/5eb4ba1f14f94bbeb63e16d4172d05dd, entries=250, sequenceid=52, filesize=16.3 K 2024-11-16T05:41:42,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/8e6cf6af52cc491487d654b686d0d6b1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/8e6cf6af52cc491487d654b686d0d6b1 2024-11-16T05:41:42,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/8e6cf6af52cc491487d654b686d0d6b1, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T05:41:42,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/3ce17a6f16244293949aba393bfc2178 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/3ce17a6f16244293949aba393bfc2178 2024-11-16T05:41:42,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/3ce17a6f16244293949aba393bfc2178, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T05:41:42,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for df3e8cf5ba630b1a9dfd75e69902d58f in 850ms, sequenceid=52, compaction requested=true 2024-11-16T05:41:42,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:42,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:42,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:42,402 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:42,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:42,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:42,402 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:42,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:42,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:42,403 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:42,403 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:42,403 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/A is initiating minor compaction (all files) 2024-11-16T05:41:42,403 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/B is initiating minor compaction (all files) 2024-11-16T05:41:42,404 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/A in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:42,404 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/B in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:42,404 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/0b6ace7d1ee240efb05cfa1442dffc54, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9c5b9774bda44120b12a0f4437784bed, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/5eb4ba1f14f94bbeb63e16d4172d05dd] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=39.7 K 2024-11-16T05:41:42,404 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/85782b287c454bd7a181c9a0a730b827, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/e3fc23a83b5a405c9e73036d7d93afe7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/8e6cf6af52cc491487d654b686d0d6b1] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=35.2 K 2024-11-16T05:41:42,404 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b6ace7d1ee240efb05cfa1442dffc54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731735698692 2024-11-16T05:41:42,404 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 85782b287c454bd7a181c9a0a730b827, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731735698692 2024-11-16T05:41:42,404 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c5b9774bda44120b12a0f4437784bed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731735698715 2024-11-16T05:41:42,404 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e3fc23a83b5a405c9e73036d7d93afe7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731735698715 2024-11-16T05:41:42,404 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eb4ba1f14f94bbeb63e16d4172d05dd, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731735700911 2024-11-16T05:41:42,404 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e6cf6af52cc491487d654b686d0d6b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731735700911 2024-11-16T05:41:42,410 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:42,410 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#A#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:42,410 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/76cf373a559c40e390a7aacf7f8dcb35 is 50, key is test_row_0/A:col10/1731735701552/Put/seqid=0 2024-11-16T05:41:42,410 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/2540807052a54efda4366da704d9e38f is 50, key is test_row_0/B:col10/1731735701552/Put/seqid=0 2024-11-16T05:41:42,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742294_1470 (size=12104) 2024-11-16T05:41:42,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742295_1471 (size=12104) 2024-11-16T05:41:42,432 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/2540807052a54efda4366da704d9e38f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/2540807052a54efda4366da704d9e38f 2024-11-16T05:41:42,435 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/B of df3e8cf5ba630b1a9dfd75e69902d58f into 2540807052a54efda4366da704d9e38f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:42,435 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:42,435 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/B, priority=13, startTime=1731735702402; duration=0sec 2024-11-16T05:41:42,435 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:42,436 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:B 2024-11-16T05:41:42,436 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:42,437 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:42,437 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/C is initiating minor compaction (all files) 2024-11-16T05:41:42,437 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/C in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:42,437 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d15f92ee08f5452ab1dc2a0b51aa05df, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/7c124dee3987478f8abdc7aa0c34ae6e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/3ce17a6f16244293949aba393bfc2178] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=35.2 K 2024-11-16T05:41:42,437 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting d15f92ee08f5452ab1dc2a0b51aa05df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731735698692 2024-11-16T05:41:42,437 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c124dee3987478f8abdc7aa0c34ae6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731735698715 2024-11-16T05:41:42,438 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ce17a6f16244293949aba393bfc2178, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731735700911 2024-11-16T05:41:42,444 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:42,444 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/6eb4164dba6442f9bbd519fd165f19cc is 50, key is test_row_0/C:col10/1731735701552/Put/seqid=0 2024-11-16T05:41:42,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742296_1472 (size=12104) 2024-11-16T05:41:42,461 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/6eb4164dba6442f9bbd519fd165f19cc as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6eb4164dba6442f9bbd519fd165f19cc 2024-11-16T05:41:42,465 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/C of df3e8cf5ba630b1a9dfd75e69902d58f into 6eb4164dba6442f9bbd519fd165f19cc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:42,465 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:42,465 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/C, priority=13, startTime=1731735702402; duration=0sec 2024-11-16T05:41:42,465 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:42,465 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:C 2024-11-16T05:41:42,606 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:41:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:42,732 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:42,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T05:41:42,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/b7075cf7eb724ef7b15b0007d1d563f7 is 50, key is test_row_0/A:col10/1731735701588/Put/seqid=0 2024-11-16T05:41:42,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742297_1473 (size=14341) 2024-11-16T05:41:42,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/b7075cf7eb724ef7b15b0007d1d563f7 2024-11-16T05:41:42,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/61ed7d807ad142758922749c7715bad8 is 50, key is test_row_0/B:col10/1731735701588/Put/seqid=0 2024-11-16T05:41:42,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742298_1474 (size=12001) 2024-11-16T05:41:42,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735762755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735762756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735762757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735762759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735762760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,833 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/76cf373a559c40e390a7aacf7f8dcb35 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/76cf373a559c40e390a7aacf7f8dcb35 2024-11-16T05:41:42,836 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/A of df3e8cf5ba630b1a9dfd75e69902d58f into 76cf373a559c40e390a7aacf7f8dcb35(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:42,836 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:42,836 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/A, priority=13, startTime=1731735702402; duration=0sec 2024-11-16T05:41:42,836 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:42,836 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:A 2024-11-16T05:41:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:41:42,848 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:42,848 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-16T05:41:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:41:42,850 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:42,850 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:42,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:42,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735762868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735762868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735762878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735762878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735762878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:41:43,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T05:41:43,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:43,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:43,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:43,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735763078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735763078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735763088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735763088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735763088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/61ed7d807ad142758922749c7715bad8 2024-11-16T05:41:43,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T05:41:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:43,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/69991208ad6b4979889c4da6a39d9e32 is 50, key is test_row_0/C:col10/1731735701588/Put/seqid=0 2024-11-16T05:41:43,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742299_1475 (size=12001) 2024-11-16T05:41:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:41:43,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T05:41:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:43,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735763388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735763388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735763398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735763399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735763399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T05:41:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:43,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:41:43,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/69991208ad6b4979889c4da6a39d9e32 2024-11-16T05:41:43,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/b7075cf7eb724ef7b15b0007d1d563f7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/b7075cf7eb724ef7b15b0007d1d563f7 2024-11-16T05:41:43,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/b7075cf7eb724ef7b15b0007d1d563f7, entries=200, sequenceid=76, filesize=14.0 K 2024-11-16T05:41:43,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/61ed7d807ad142758922749c7715bad8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/61ed7d807ad142758922749c7715bad8 2024-11-16T05:41:43,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/61ed7d807ad142758922749c7715bad8, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T05:41:43,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/69991208ad6b4979889c4da6a39d9e32 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/69991208ad6b4979889c4da6a39d9e32 2024-11-16T05:41:43,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/69991208ad6b4979889c4da6a39d9e32, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T05:41:43,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for df3e8cf5ba630b1a9dfd75e69902d58f in 845ms, sequenceid=76, compaction requested=false 2024-11-16T05:41:43,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:43,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T05:41:43,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:43,614 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T05:41:43,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d5731d307bf3445fad22b36e585fa60a is 50, key is test_row_0/A:col10/1731735702758/Put/seqid=0 2024-11-16T05:41:43,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742300_1476 (size=12001) 2024-11-16T05:41:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:43,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735763936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735763937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735763938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735763953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735763953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:41:44,023 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d5731d307bf3445fad22b36e585fa60a 2024-11-16T05:41:44,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/ef20351037ee478f948e48f2ec6d2d25 is 50, key is test_row_0/B:col10/1731735702758/Put/seqid=0 2024-11-16T05:41:44,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742301_1477 (size=12001) 2024-11-16T05:41:44,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735764058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735764058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735764058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735764068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735764068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735764268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735764268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735764268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735764278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735764288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,432 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/ef20351037ee478f948e48f2ec6d2d25 2024-11-16T05:41:44,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/a4385621bc6a4897beb885b49ce1ebe0 is 50, key is test_row_0/C:col10/1731735702758/Put/seqid=0 2024-11-16T05:41:44,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742302_1478 (size=12001) 2024-11-16T05:41:44,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735764578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735764578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735764578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735764588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735764598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:44,841 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/a4385621bc6a4897beb885b49ce1ebe0 2024-11-16T05:41:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d5731d307bf3445fad22b36e585fa60a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d5731d307bf3445fad22b36e585fa60a 2024-11-16T05:41:44,847 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d5731d307bf3445fad22b36e585fa60a, entries=150, sequenceid=91, filesize=11.7 K 2024-11-16T05:41:44,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/ef20351037ee478f948e48f2ec6d2d25 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/ef20351037ee478f948e48f2ec6d2d25 2024-11-16T05:41:44,851 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/ef20351037ee478f948e48f2ec6d2d25, entries=150, sequenceid=91, filesize=11.7 K 2024-11-16T05:41:44,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/a4385621bc6a4897beb885b49ce1ebe0 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a4385621bc6a4897beb885b49ce1ebe0 2024-11-16T05:41:44,854 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a4385621bc6a4897beb885b49ce1ebe0, entries=150, sequenceid=91, filesize=11.7 K 2024-11-16T05:41:44,855 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for df3e8cf5ba630b1a9dfd75e69902d58f in 1241ms, sequenceid=91, compaction requested=true 2024-11-16T05:41:44,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:44,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:44,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-16T05:41:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-16T05:41:44,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-16T05:41:44,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0060 sec 2024-11-16T05:41:44,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.0100 sec 2024-11-16T05:41:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:41:44,998 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:44,999 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-16T05:41:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T05:41:45,001 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:45,001 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:45,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:45,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T05:41:45,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/2b0d578c20b74e4eaa1ad74e70e4812b is 50, key is test_row_0/A:col10/1731735705090/Put/seqid=0 2024-11-16T05:41:45,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742303_1479 (size=14341) 2024-11-16T05:41:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T05:41:45,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735765107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735765108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735765112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735765118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735765119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T05:41:45,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:45,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735765228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735765228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735765228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735765238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T05:41:45,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:45,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T05:41:45,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735765438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735765438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735765438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735765448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T05:41:45,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:45,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/2b0d578c20b74e4eaa1ad74e70e4812b 2024-11-16T05:41:45,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/c834f64135874bddbc14c4c50c5319ed is 50, key is test_row_0/B:col10/1731735705090/Put/seqid=0 2024-11-16T05:41:45,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742304_1480 (size=12001) 2024-11-16T05:41:45,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T05:41:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T05:41:45,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735765748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735765749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735765749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T05:41:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735765758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:45,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/c834f64135874bddbc14c4c50c5319ed 2024-11-16T05:41:45,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T05:41:45,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:45,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:45,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:45,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/dd86a1dcceed4796bb8d6f33bbdf6cbd is 50, key is test_row_0/C:col10/1731735705090/Put/seqid=0 2024-11-16T05:41:45,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742305_1481 (size=12001) 2024-11-16T05:41:46,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T05:41:46,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:46,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:46,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735766128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:46,134 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in df3e8cf5ba630b1a9dfd75e69902d58f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:46,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T05:41:46,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T05:41:46,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:46,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:46,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:46,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:46,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735766268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735766268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735766268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:46,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735766278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:46,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/dd86a1dcceed4796bb8d6f33bbdf6cbd 2024-11-16T05:41:46,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/2b0d578c20b74e4eaa1ad74e70e4812b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/2b0d578c20b74e4eaa1ad74e70e4812b 2024-11-16T05:41:46,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/2b0d578c20b74e4eaa1ad74e70e4812b, entries=200, sequenceid=114, filesize=14.0 K 2024-11-16T05:41:46,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/c834f64135874bddbc14c4c50c5319ed as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/c834f64135874bddbc14c4c50c5319ed 2024-11-16T05:41:46,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/c834f64135874bddbc14c4c50c5319ed, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T05:41:46,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/dd86a1dcceed4796bb8d6f33bbdf6cbd as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/dd86a1dcceed4796bb8d6f33bbdf6cbd 2024-11-16T05:41:46,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/dd86a1dcceed4796bb8d6f33bbdf6cbd, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T05:41:46,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for df3e8cf5ba630b1a9dfd75e69902d58f in 1250ms, sequenceid=114, compaction requested=true 2024-11-16T05:41:46,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:46,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:46,341 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:41:46,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:46,341 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:41:46,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:46,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:46,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:46,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:46,342 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:41:46,342 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:41:46,342 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/B is initiating minor compaction (all files) 2024-11-16T05:41:46,342 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/A is initiating minor compaction (all files) 2024-11-16T05:41:46,342 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/A in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:46,342 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/B in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:46,342 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/76cf373a559c40e390a7aacf7f8dcb35, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/b7075cf7eb724ef7b15b0007d1d563f7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d5731d307bf3445fad22b36e585fa60a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/2b0d578c20b74e4eaa1ad74e70e4812b] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=51.5 K 2024-11-16T05:41:46,342 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/2540807052a54efda4366da704d9e38f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/61ed7d807ad142758922749c7715bad8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/ef20351037ee478f948e48f2ec6d2d25, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/c834f64135874bddbc14c4c50c5319ed] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=47.0 K 2024-11-16T05:41:46,342 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2540807052a54efda4366da704d9e38f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731735700911 2024-11-16T05:41:46,342 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76cf373a559c40e390a7aacf7f8dcb35, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731735700911 2024-11-16T05:41:46,343 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 61ed7d807ad142758922749c7715bad8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731735701588 2024-11-16T05:41:46,343 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7075cf7eb724ef7b15b0007d1d563f7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731735701588 2024-11-16T05:41:46,343 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting ef20351037ee478f948e48f2ec6d2d25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731735702746 2024-11-16T05:41:46,343 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5731d307bf3445fad22b36e585fa60a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731735702746 2024-11-16T05:41:46,343 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting c834f64135874bddbc14c4c50c5319ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731735703951 2024-11-16T05:41:46,343 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b0d578c20b74e4eaa1ad74e70e4812b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731735703932 2024-11-16T05:41:46,349 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:46,350 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/f7fdde764a65404a98072a46b8bf7ba2 is 50, key is test_row_0/B:col10/1731735705090/Put/seqid=0 2024-11-16T05:41:46,353 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:46,354 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/c79134bcff914b8290708634bb18058a is 50, key is test_row_0/A:col10/1731735705090/Put/seqid=0 2024-11-16T05:41:46,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742306_1482 (size=12241) 2024-11-16T05:41:46,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742307_1483 (size=12241) 2024-11-16T05:41:46,361 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/c79134bcff914b8290708634bb18058a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/c79134bcff914b8290708634bb18058a 2024-11-16T05:41:46,367 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/A of df3e8cf5ba630b1a9dfd75e69902d58f into c79134bcff914b8290708634bb18058a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:46,367 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:46,367 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/A, priority=12, startTime=1731735706341; duration=0sec 2024-11-16T05:41:46,367 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:46,367 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:A 2024-11-16T05:41:46,367 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:41:46,369 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:41:46,369 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/C is initiating minor compaction (all files) 2024-11-16T05:41:46,369 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/C in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:46,369 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6eb4164dba6442f9bbd519fd165f19cc, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/69991208ad6b4979889c4da6a39d9e32, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a4385621bc6a4897beb885b49ce1ebe0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/dd86a1dcceed4796bb8d6f33bbdf6cbd] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=47.0 K 2024-11-16T05:41:46,369 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eb4164dba6442f9bbd519fd165f19cc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731735700911 2024-11-16T05:41:46,369 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69991208ad6b4979889c4da6a39d9e32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731735701588 2024-11-16T05:41:46,370 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4385621bc6a4897beb885b49ce1ebe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731735702746 2024-11-16T05:41:46,370 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd86a1dcceed4796bb8d6f33bbdf6cbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731735703951 2024-11-16T05:41:46,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T05:41:46,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:46,375 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T05:41:46,376 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#C#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:46,376 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/166367fb47bf40b8ac20a32c2f3562ea is 50, key is test_row_0/C:col10/1731735705090/Put/seqid=0 2024-11-16T05:41:46,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/abee94c1d2b24b91b50408f7c46ed296 is 50, key is test_row_0/A:col10/1731735705116/Put/seqid=0 2024-11-16T05:41:46,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742308_1484 (size=12241) 2024-11-16T05:41:46,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742309_1485 (size=12001) 2024-11-16T05:41:46,760 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/f7fdde764a65404a98072a46b8bf7ba2 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/f7fdde764a65404a98072a46b8bf7ba2 2024-11-16T05:41:46,764 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/B of df3e8cf5ba630b1a9dfd75e69902d58f into f7fdde764a65404a98072a46b8bf7ba2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:46,764 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:46,764 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/B, priority=12, startTime=1731735706341; duration=0sec 2024-11-16T05:41:46,764 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:46,764 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:B 2024-11-16T05:41:46,783 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/abee94c1d2b24b91b50408f7c46ed296 2024-11-16T05:41:46,786 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/166367fb47bf40b8ac20a32c2f3562ea as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/166367fb47bf40b8ac20a32c2f3562ea 2024-11-16T05:41:46,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/667b96969f544459a39bf05e31fd51ec is 50, key is test_row_0/B:col10/1731735705116/Put/seqid=0 2024-11-16T05:41:46,794 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/C of df3e8cf5ba630b1a9dfd75e69902d58f into 166367fb47bf40b8ac20a32c2f3562ea(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:46,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742310_1486 (size=12001) 2024-11-16T05:41:46,794 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:46,794 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/C, priority=12, startTime=1731735706341; duration=0sec 2024-11-16T05:41:46,794 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:46,794 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:C 2024-11-16T05:41:46,796 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/667b96969f544459a39bf05e31fd51ec 2024-11-16T05:41:46,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/030ab0fd6b574d8ab69b23d2e90978b7 is 50, key is test_row_0/C:col10/1731735705116/Put/seqid=0 2024-11-16T05:41:46,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742311_1487 (size=12001) 2024-11-16T05:41:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T05:41:47,206 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/030ab0fd6b574d8ab69b23d2e90978b7 2024-11-16T05:41:47,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/abee94c1d2b24b91b50408f7c46ed296 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/abee94c1d2b24b91b50408f7c46ed296 2024-11-16T05:41:47,213 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/abee94c1d2b24b91b50408f7c46ed296, entries=150, sequenceid=128, filesize=11.7 K 2024-11-16T05:41:47,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/667b96969f544459a39bf05e31fd51ec as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/667b96969f544459a39bf05e31fd51ec 2024-11-16T05:41:47,217 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/667b96969f544459a39bf05e31fd51ec, entries=150, sequenceid=128, filesize=11.7 K 2024-11-16T05:41:47,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/030ab0fd6b574d8ab69b23d2e90978b7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/030ab0fd6b574d8ab69b23d2e90978b7 2024-11-16T05:41:47,221 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/030ab0fd6b574d8ab69b23d2e90978b7, entries=150, sequenceid=128, filesize=11.7 K 2024-11-16T05:41:47,222 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for df3e8cf5ba630b1a9dfd75e69902d58f in 847ms, sequenceid=128, compaction requested=false 2024-11-16T05:41:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-16T05:41:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-16T05:41:47,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-16T05:41:47,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2220 sec 2024-11-16T05:41:47,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.2260 sec 2024-11-16T05:41:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:47,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:41:47,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/bc201d0e6e654435b71b2e9f4df0370f is 50, key is test_row_0/A:col10/1731735707293/Put/seqid=0 2024-11-16T05:41:47,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742312_1488 (size=19321) 2024-11-16T05:41:47,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/bc201d0e6e654435b71b2e9f4df0370f 2024-11-16T05:41:47,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/9adb1992658247e597b37b118eccea14 is 50, key is test_row_0/B:col10/1731735707293/Put/seqid=0 2024-11-16T05:41:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742313_1489 (size=12151) 2024-11-16T05:41:47,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735767370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735767370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735767370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735767371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735767488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735767488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735767488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735767488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735767698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735767698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735767698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735767698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:47,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/9adb1992658247e597b37b118eccea14 2024-11-16T05:41:47,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/d5b9ad59c4e74db4bcc8c26bff7d2183 is 50, key is test_row_0/C:col10/1731735707293/Put/seqid=0 2024-11-16T05:41:47,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742314_1490 (size=12151) 2024-11-16T05:41:48,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735768018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735768018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735768018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735768018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735768148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,155 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in df3e8cf5ba630b1a9dfd75e69902d58f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/d5b9ad59c4e74db4bcc8c26bff7d2183 2024-11-16T05:41:48,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/bc201d0e6e654435b71b2e9f4df0370f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/bc201d0e6e654435b71b2e9f4df0370f 2024-11-16T05:41:48,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/bc201d0e6e654435b71b2e9f4df0370f, entries=300, sequenceid=141, filesize=18.9 K 2024-11-16T05:41:48,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/9adb1992658247e597b37b118eccea14 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/9adb1992658247e597b37b118eccea14 2024-11-16T05:41:48,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/9adb1992658247e597b37b118eccea14, entries=150, sequenceid=141, filesize=11.9 K 2024-11-16T05:41:48,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/d5b9ad59c4e74db4bcc8c26bff7d2183 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d5b9ad59c4e74db4bcc8c26bff7d2183 2024-11-16T05:41:48,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d5b9ad59c4e74db4bcc8c26bff7d2183, entries=150, sequenceid=141, filesize=11.9 K 2024-11-16T05:41:48,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for df3e8cf5ba630b1a9dfd75e69902d58f in 876ms, sequenceid=141, compaction requested=true 2024-11-16T05:41:48,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:48,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:48,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:48,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:48,174 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:48,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:48,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:48,174 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:48,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:48,174 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43563 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:48,174 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:48,174 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/A is initiating minor compaction (all files) 2024-11-16T05:41:48,174 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/B is initiating minor compaction (all files) 2024-11-16T05:41:48,174 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/A in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:48,174 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/B in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:48,174 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/c79134bcff914b8290708634bb18058a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/abee94c1d2b24b91b50408f7c46ed296, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/bc201d0e6e654435b71b2e9f4df0370f] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=42.5 K 2024-11-16T05:41:48,174 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/f7fdde764a65404a98072a46b8bf7ba2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/667b96969f544459a39bf05e31fd51ec, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/9adb1992658247e597b37b118eccea14] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=35.5 K 2024-11-16T05:41:48,175 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c79134bcff914b8290708634bb18058a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731735703951 2024-11-16T05:41:48,175 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f7fdde764a65404a98072a46b8bf7ba2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731735703951 2024-11-16T05:41:48,175 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting abee94c1d2b24b91b50408f7c46ed296, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731735705111 2024-11-16T05:41:48,175 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 667b96969f544459a39bf05e31fd51ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731735705111 2024-11-16T05:41:48,176 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc201d0e6e654435b71b2e9f4df0370f, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731735707290 2024-11-16T05:41:48,176 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9adb1992658247e597b37b118eccea14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731735707293 2024-11-16T05:41:48,181 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#B#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:48,182 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/70bb8d9d91d54bf48745fcfd3dba571b is 50, key is test_row_0/B:col10/1731735707293/Put/seqid=0 2024-11-16T05:41:48,184 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:48,185 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/775b9392e0d243a2aac0f5fc0832ea57 is 50, key is test_row_0/A:col10/1731735707293/Put/seqid=0 2024-11-16T05:41:48,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742316_1492 (size=12493) 2024-11-16T05:41:48,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742315_1491 (size=12493) 2024-11-16T05:41:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:48,533 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:48,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T05:41:48,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/7714fa5c042943188da10f379f04915c is 50, key is test_row_0/A:col10/1731735707350/Put/seqid=0 2024-11-16T05:41:48,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742317_1493 (size=14541) 2024-11-16T05:41:48,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735768544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735768545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735768549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735768557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,598 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/775b9392e0d243a2aac0f5fc0832ea57 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/775b9392e0d243a2aac0f5fc0832ea57 2024-11-16T05:41:48,598 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/70bb8d9d91d54bf48745fcfd3dba571b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/70bb8d9d91d54bf48745fcfd3dba571b 2024-11-16T05:41:48,601 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/A of df3e8cf5ba630b1a9dfd75e69902d58f into 775b9392e0d243a2aac0f5fc0832ea57(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:48,601 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/B of df3e8cf5ba630b1a9dfd75e69902d58f into 70bb8d9d91d54bf48745fcfd3dba571b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:48,601 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:48,601 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:48,601 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/B, priority=13, startTime=1731735708174; duration=0sec 2024-11-16T05:41:48,601 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/A, priority=13, startTime=1731735708174; duration=0sec 2024-11-16T05:41:48,601 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:48,601 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:B 2024-11-16T05:41:48,601 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:48,601 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:A 2024-11-16T05:41:48,601 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:48,602 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:48,602 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/C is initiating minor compaction (all files) 2024-11-16T05:41:48,602 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/C in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:48,602 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/166367fb47bf40b8ac20a32c2f3562ea, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/030ab0fd6b574d8ab69b23d2e90978b7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d5b9ad59c4e74db4bcc8c26bff7d2183] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=35.5 K 2024-11-16T05:41:48,602 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 166367fb47bf40b8ac20a32c2f3562ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731735703951 2024-11-16T05:41:48,602 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 030ab0fd6b574d8ab69b23d2e90978b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731735705111 2024-11-16T05:41:48,602 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b9ad59c4e74db4bcc8c26bff7d2183, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731735707293 2024-11-16T05:41:48,607 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#C#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:48,607 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/8bdcd575d04a4e69b482aadf53053903 is 50, key is test_row_0/C:col10/1731735707293/Put/seqid=0 2024-11-16T05:41:48,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742318_1494 (size=12493) 2024-11-16T05:41:48,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735768667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735768668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735768668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735768668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735768878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735768878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735768878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:48,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735768878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:48,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:48,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:48,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/7714fa5c042943188da10f379f04915c 2024-11-16T05:41:48,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/0f3a9804f8a242828ccc31ce01a5b29e is 50, key is test_row_0/B:col10/1731735707350/Put/seqid=0 2024-11-16T05:41:48,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742319_1495 (size=12151) 2024-11-16T05:41:49,014 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/8bdcd575d04a4e69b482aadf53053903 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8bdcd575d04a4e69b482aadf53053903 2024-11-16T05:41:49,017 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/C of df3e8cf5ba630b1a9dfd75e69902d58f into 8bdcd575d04a4e69b482aadf53053903(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:49,017 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:49,017 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/C, priority=13, startTime=1731735708174; duration=0sec 2024-11-16T05:41:49,017 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:49,017 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:C 2024-11-16T05:41:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T05:41:49,169 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:49,169 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-16T05:41:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T05:41:49,170 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:49,171 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:49,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:49,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735769188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735769188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735769188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735769188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T05:41:49,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T05:41:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:49,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:49,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:49,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:49,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/0f3a9804f8a242828ccc31ce01a5b29e 2024-11-16T05:41:49,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/860253e544e54e378b08b83972dd7b66 is 50, key is test_row_0/C:col10/1731735707350/Put/seqid=0 2024-11-16T05:41:49,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742320_1496 (size=12151) 2024-11-16T05:41:49,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/860253e544e54e378b08b83972dd7b66 2024-11-16T05:41:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/7714fa5c042943188da10f379f04915c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/7714fa5c042943188da10f379f04915c 2024-11-16T05:41:49,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/7714fa5c042943188da10f379f04915c, entries=200, sequenceid=167, filesize=14.2 K 2024-11-16T05:41:49,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/0f3a9804f8a242828ccc31ce01a5b29e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/0f3a9804f8a242828ccc31ce01a5b29e 2024-11-16T05:41:49,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/0f3a9804f8a242828ccc31ce01a5b29e, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T05:41:49,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/860253e544e54e378b08b83972dd7b66 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/860253e544e54e378b08b83972dd7b66 2024-11-16T05:41:49,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/860253e544e54e378b08b83972dd7b66, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T05:41:49,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for df3e8cf5ba630b1a9dfd75e69902d58f in 849ms, sequenceid=167, compaction requested=false 2024-11-16T05:41:49,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:49,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T05:41:49,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:49,476 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T05:41:49,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/83b91edd9b404480b9bc730fe0d570c4 is 50, key is test_row_0/A:col10/1731735708548/Put/seqid=0 2024-11-16T05:41:49,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742321_1497 (size=12151) 2024-11-16T05:41:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T05:41:49,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735769745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735769746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735769747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735769748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T05:41:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735769858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735769858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735769868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735769868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:49,884 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/83b91edd9b404480b9bc730fe0d570c4 2024-11-16T05:41:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/6c44b3b941f94918b3ad4584f45ce6c7 is 50, key is test_row_0/B:col10/1731735708548/Put/seqid=0 2024-11-16T05:41:49,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742322_1498 (size=12151) 2024-11-16T05:41:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735770068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735770068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735770078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735770078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,294 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/6c44b3b941f94918b3ad4584f45ce6c7 2024-11-16T05:41:50,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/6e6a886be2f24810b100c5243c00a449 is 50, key is test_row_0/C:col10/1731735708548/Put/seqid=0 2024-11-16T05:41:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742323_1499 (size=12151) 2024-11-16T05:41:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T05:41:50,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735770378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735770378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735770388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735770388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,704 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/6e6a886be2f24810b100c5243c00a449 2024-11-16T05:41:50,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/83b91edd9b404480b9bc730fe0d570c4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/83b91edd9b404480b9bc730fe0d570c4 2024-11-16T05:41:50,710 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/83b91edd9b404480b9bc730fe0d570c4, entries=150, sequenceid=180, filesize=11.9 K 2024-11-16T05:41:50,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/6c44b3b941f94918b3ad4584f45ce6c7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/6c44b3b941f94918b3ad4584f45ce6c7 2024-11-16T05:41:50,713 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/6c44b3b941f94918b3ad4584f45ce6c7, entries=150, sequenceid=180, filesize=11.9 K 2024-11-16T05:41:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/6e6a886be2f24810b100c5243c00a449 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6e6a886be2f24810b100c5243c00a449 2024-11-16T05:41:50,717 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6e6a886be2f24810b100c5243c00a449, entries=150, sequenceid=180, filesize=11.9 K 2024-11-16T05:41:50,718 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for df3e8cf5ba630b1a9dfd75e69902d58f in 1241ms, sequenceid=180, compaction requested=true 2024-11-16T05:41:50,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:50,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:50,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-16T05:41:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-16T05:41:50,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-16T05:41:50,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5480 sec 2024-11-16T05:41:50,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.5510 sec 2024-11-16T05:41:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:50,901 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:50,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T05:41:50,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d54147aab5d64a68b0a4c1f62293d8c3 is 50, key is test_row_0/A:col10/1731735709747/Put/seqid=0 2024-11-16T05:41:50,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742324_1500 (size=14541) 2024-11-16T05:41:50,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d54147aab5d64a68b0a4c1f62293d8c3 2024-11-16T05:41:50,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/3a68993d158e47bc9c208bfce0328b35 is 50, key is test_row_0/B:col10/1731735709747/Put/seqid=0 2024-11-16T05:41:50,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742325_1501 (size=12151) 2024-11-16T05:41:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735770914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735770916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/3a68993d158e47bc9c208bfce0328b35 2024-11-16T05:41:50,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735770917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735770918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:50,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/5e9ff20b4bb646fb8d602579728f07d3 is 50, key is test_row_0/C:col10/1731735709747/Put/seqid=0 2024-11-16T05:41:50,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742326_1502 (size=12151) 2024-11-16T05:41:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735771028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735771028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735771038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735771038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735771238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735771238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735771248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735771248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T05:41:51,318 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:51,319 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-16T05:41:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T05:41:51,320 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:51,320 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:51,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:51,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/5e9ff20b4bb646fb8d602579728f07d3 2024-11-16T05:41:51,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d54147aab5d64a68b0a4c1f62293d8c3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d54147aab5d64a68b0a4c1f62293d8c3 2024-11-16T05:41:51,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d54147aab5d64a68b0a4c1f62293d8c3, entries=200, sequenceid=207, filesize=14.2 K 2024-11-16T05:41:51,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/3a68993d158e47bc9c208bfce0328b35 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/3a68993d158e47bc9c208bfce0328b35 2024-11-16T05:41:51,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/3a68993d158e47bc9c208bfce0328b35, entries=150, sequenceid=207, filesize=11.9 K 2024-11-16T05:41:51,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/5e9ff20b4bb646fb8d602579728f07d3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/5e9ff20b4bb646fb8d602579728f07d3 2024-11-16T05:41:51,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/5e9ff20b4bb646fb8d602579728f07d3, entries=150, sequenceid=207, filesize=11.9 K 2024-11-16T05:41:51,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for df3e8cf5ba630b1a9dfd75e69902d58f in 447ms, sequenceid=207, compaction requested=true 2024-11-16T05:41:51,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:51,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:51,348 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:41:51,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:51,348 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:41:51,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:51,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:51,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:51,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:51,349 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:41:51,349 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:41:51,349 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/A is initiating minor compaction (all files) 2024-11-16T05:41:51,349 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/B is initiating minor compaction (all files) 2024-11-16T05:41:51,349 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/A in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:51,349 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/B in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:51,349 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/775b9392e0d243a2aac0f5fc0832ea57, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/7714fa5c042943188da10f379f04915c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/83b91edd9b404480b9bc730fe0d570c4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d54147aab5d64a68b0a4c1f62293d8c3] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=52.5 K 2024-11-16T05:41:51,349 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/70bb8d9d91d54bf48745fcfd3dba571b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/0f3a9804f8a242828ccc31ce01a5b29e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/6c44b3b941f94918b3ad4584f45ce6c7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/3a68993d158e47bc9c208bfce0328b35] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=47.8 K 2024-11-16T05:41:51,349 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 775b9392e0d243a2aac0f5fc0832ea57, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731735707293 2024-11-16T05:41:51,349 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 70bb8d9d91d54bf48745fcfd3dba571b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731735707293 2024-11-16T05:41:51,349 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7714fa5c042943188da10f379f04915c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731735707331 2024-11-16T05:41:51,349 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f3a9804f8a242828ccc31ce01a5b29e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731735707350 2024-11-16T05:41:51,349 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83b91edd9b404480b9bc730fe0d570c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731735708542 2024-11-16T05:41:51,349 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c44b3b941f94918b3ad4584f45ce6c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731735708542 2024-11-16T05:41:51,350 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d54147aab5d64a68b0a4c1f62293d8c3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731735709736 2024-11-16T05:41:51,350 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a68993d158e47bc9c208bfce0328b35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731735709736 2024-11-16T05:41:51,356 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:51,356 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/75d226c0009141acafe3e97a45d78339 is 50, key is test_row_0/B:col10/1731735709747/Put/seqid=0 2024-11-16T05:41:51,356 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#A#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:51,356 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/dde3efc5fe3a4fb88a70a731057f451f is 50, key is test_row_0/A:col10/1731735709747/Put/seqid=0 2024-11-16T05:41:51,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742327_1503 (size=12629) 2024-11-16T05:41:51,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742328_1504 (size=12629) 2024-11-16T05:41:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T05:41:51,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T05:41:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:51,471 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T05:41:51,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d41536e070444b75a9b88bf7fdb8eb63 is 50, key is test_row_0/A:col10/1731735710916/Put/seqid=0 2024-11-16T05:41:51,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742329_1505 (size=9757) 2024-11-16T05:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:51,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:51,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735771610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735771611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735771612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735771618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T05:41:51,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735771728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735771728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735771728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735771728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,770 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/75d226c0009141acafe3e97a45d78339 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/75d226c0009141acafe3e97a45d78339 2024-11-16T05:41:51,770 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/dde3efc5fe3a4fb88a70a731057f451f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/dde3efc5fe3a4fb88a70a731057f451f 2024-11-16T05:41:51,774 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/A of df3e8cf5ba630b1a9dfd75e69902d58f into dde3efc5fe3a4fb88a70a731057f451f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:51,774 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/B of df3e8cf5ba630b1a9dfd75e69902d58f into 75d226c0009141acafe3e97a45d78339(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:51,774 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:51,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:51,774 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/A, priority=12, startTime=1731735711348; duration=0sec 2024-11-16T05:41:51,774 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/B, priority=12, startTime=1731735711348; duration=0sec 2024-11-16T05:41:51,774 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:51,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:51,774 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:A 2024-11-16T05:41:51,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:B 2024-11-16T05:41:51,774 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:41:51,775 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:41:51,775 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/C is initiating minor compaction (all files) 2024-11-16T05:41:51,775 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/C in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:51,775 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8bdcd575d04a4e69b482aadf53053903, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/860253e544e54e378b08b83972dd7b66, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6e6a886be2f24810b100c5243c00a449, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/5e9ff20b4bb646fb8d602579728f07d3] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=47.8 K 2024-11-16T05:41:51,775 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bdcd575d04a4e69b482aadf53053903, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731735707293 2024-11-16T05:41:51,775 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 860253e544e54e378b08b83972dd7b66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731735707350 2024-11-16T05:41:51,775 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6a886be2f24810b100c5243c00a449, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731735708542 2024-11-16T05:41:51,776 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e9ff20b4bb646fb8d602579728f07d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731735709736 2024-11-16T05:41:51,781 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:51,782 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/89e1312305314003a40f4a9f35c00cf7 is 50, key is test_row_0/C:col10/1731735709747/Put/seqid=0 2024-11-16T05:41:51,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742330_1506 (size=12629) 2024-11-16T05:41:51,791 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/89e1312305314003a40f4a9f35c00cf7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/89e1312305314003a40f4a9f35c00cf7 2024-11-16T05:41:51,794 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/C of df3e8cf5ba630b1a9dfd75e69902d58f into 89e1312305314003a40f4a9f35c00cf7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:51,794 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:51,794 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/C, priority=12, startTime=1731735711348; duration=0sec 2024-11-16T05:41:51,794 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:51,794 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:C 2024-11-16T05:41:51,878 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d41536e070444b75a9b88bf7fdb8eb63 2024-11-16T05:41:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/fb7714ce4d564a8bbe45a97947a537c7 is 50, key is test_row_0/B:col10/1731735710916/Put/seqid=0 2024-11-16T05:41:51,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742331_1507 (size=9757) 2024-11-16T05:41:51,890 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/fb7714ce4d564a8bbe45a97947a537c7 2024-11-16T05:41:51,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/a675398464cc4767935e54cd382a61eb is 50, key is test_row_0/C:col10/1731735710916/Put/seqid=0 2024-11-16T05:41:51,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742332_1508 (size=9757) 2024-11-16T05:41:51,907 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/a675398464cc4767935e54cd382a61eb 2024-11-16T05:41:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d41536e070444b75a9b88bf7fdb8eb63 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d41536e070444b75a9b88bf7fdb8eb63 2024-11-16T05:41:51,915 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d41536e070444b75a9b88bf7fdb8eb63, entries=100, sequenceid=216, filesize=9.5 K 2024-11-16T05:41:51,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/fb7714ce4d564a8bbe45a97947a537c7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fb7714ce4d564a8bbe45a97947a537c7 2024-11-16T05:41:51,918 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fb7714ce4d564a8bbe45a97947a537c7, entries=100, sequenceid=216, filesize=9.5 K 2024-11-16T05:41:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/a675398464cc4767935e54cd382a61eb as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a675398464cc4767935e54cd382a61eb 2024-11-16T05:41:51,923 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a675398464cc4767935e54cd382a61eb, entries=100, sequenceid=216, filesize=9.5 K 2024-11-16T05:41:51,924 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for df3e8cf5ba630b1a9dfd75e69902d58f in 452ms, sequenceid=216, compaction requested=false 2024-11-16T05:41:51,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:51,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:51,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-16T05:41:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-16T05:41:51,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-16T05:41:51,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 605 msec 2024-11-16T05:41:51,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 608 msec 2024-11-16T05:41:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T05:41:51,948 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:51,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-16T05:41:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T05:41:51,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:51,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:51,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T05:41:51,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:51,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/ab0606bdd1fc494daa9373b2b442bdbc is 50, key is test_row_0/A:col10/1731735711609/Put/seqid=0 2024-11-16T05:41:51,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742333_1509 (size=14541) 2024-11-16T05:41:51,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735771958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735771963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735771963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:51,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735771964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T05:41:52,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735772068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735772068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735772069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735772078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T05:41:52,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:52,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44924 deadline: 1731735772178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,183 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in df3e8cf5ba630b1a9dfd75e69902d58f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T05:41:52,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:52,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T05:41:52,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735772278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735772278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735772278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735772288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/ab0606bdd1fc494daa9373b2b442bdbc 2024-11-16T05:41:52,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/1d8a3136c0b94ce3a25f874e5487ea61 is 50, key is test_row_0/B:col10/1731735711609/Put/seqid=0 2024-11-16T05:41:52,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742334_1510 (size=12151) 2024-11-16T05:41:52,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T05:41:52,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:52,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T05:41:52,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:52,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T05:41:52,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735772588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735772589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735772589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735772598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:52,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T05:41:52,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:52,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:52,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/1d8a3136c0b94ce3a25f874e5487ea61 2024-11-16T05:41:52,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/32b7cac3111245ebb87b826395bee70d is 50, key is test_row_0/C:col10/1731735711609/Put/seqid=0 2024-11-16T05:41:52,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742335_1511 (size=12151) 2024-11-16T05:41:52,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/32b7cac3111245ebb87b826395bee70d 2024-11-16T05:41:52,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/ab0606bdd1fc494daa9373b2b442bdbc as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/ab0606bdd1fc494daa9373b2b442bdbc 2024-11-16T05:41:52,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/ab0606bdd1fc494daa9373b2b442bdbc, entries=200, sequenceid=247, filesize=14.2 K 2024-11-16T05:41:52,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/1d8a3136c0b94ce3a25f874e5487ea61 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/1d8a3136c0b94ce3a25f874e5487ea61 2024-11-16T05:41:52,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/1d8a3136c0b94ce3a25f874e5487ea61, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T05:41:52,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/32b7cac3111245ebb87b826395bee70d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/32b7cac3111245ebb87b826395bee70d 2024-11-16T05:41:52,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/32b7cac3111245ebb87b826395bee70d, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T05:41:52,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for df3e8cf5ba630b1a9dfd75e69902d58f in 848ms, sequenceid=247, compaction requested=true 2024-11-16T05:41:52,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:52,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:52,800 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:52,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:52,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:52,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:52,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:52,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:41:52,800 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:52,801 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36927 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:52,801 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/A is initiating minor compaction (all files) 2024-11-16T05:41:52,801 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/A in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,801 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/dde3efc5fe3a4fb88a70a731057f451f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d41536e070444b75a9b88bf7fdb8eb63, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/ab0606bdd1fc494daa9373b2b442bdbc] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=36.1 K 2024-11-16T05:41:52,801 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting dde3efc5fe3a4fb88a70a731057f451f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731735709736 2024-11-16T05:41:52,802 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d41536e070444b75a9b88bf7fdb8eb63, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731735710916 2024-11-16T05:41:52,802 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:52,802 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/B is initiating minor compaction (all files) 2024-11-16T05:41:52,802 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/B in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,802 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/75d226c0009141acafe3e97a45d78339, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fb7714ce4d564a8bbe45a97947a537c7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/1d8a3136c0b94ce3a25f874e5487ea61] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=33.7 K 2024-11-16T05:41:52,803 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab0606bdd1fc494daa9373b2b442bdbc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731735711606 2024-11-16T05:41:52,803 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 75d226c0009141acafe3e97a45d78339, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731735709736 2024-11-16T05:41:52,803 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting fb7714ce4d564a8bbe45a97947a537c7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731735710916 2024-11-16T05:41:52,803 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d8a3136c0b94ce3a25f874e5487ea61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731735711606 2024-11-16T05:41:52,820 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:52,820 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#A#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:52,820 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/419a6c777d9a4454a662e99b39cab80c is 50, key is test_row_0/B:col10/1731735711609/Put/seqid=0 2024-11-16T05:41:52,820 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/f8478beaaa5e4ccc8d267db90d4d6e2f is 50, key is test_row_0/A:col10/1731735711609/Put/seqid=0 2024-11-16T05:41:52,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742336_1512 (size=12731) 2024-11-16T05:41:52,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742337_1513 (size=12731) 2024-11-16T05:41:52,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T05:41:52,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:52,868 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T05:41:52,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d12aeb06061e40a7ae5c3b8874b86c4f is 50, key is test_row_0/A:col10/1731735711961/Put/seqid=0 2024-11-16T05:41:52,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742338_1514 (size=9757) 2024-11-16T05:41:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T05:41:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:53,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:53,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735773145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735773146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735773150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735773155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,241 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/419a6c777d9a4454a662e99b39cab80c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/419a6c777d9a4454a662e99b39cab80c 2024-11-16T05:41:53,242 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/f8478beaaa5e4ccc8d267db90d4d6e2f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/f8478beaaa5e4ccc8d267db90d4d6e2f 2024-11-16T05:41:53,244 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/B of df3e8cf5ba630b1a9dfd75e69902d58f into 419a6c777d9a4454a662e99b39cab80c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:53,244 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:53,244 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/A of df3e8cf5ba630b1a9dfd75e69902d58f into f8478beaaa5e4ccc8d267db90d4d6e2f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:53,244 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/B, priority=13, startTime=1731735712800; duration=0sec 2024-11-16T05:41:53,244 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:53,244 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/A, priority=13, startTime=1731735712800; duration=0sec 2024-11-16T05:41:53,244 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:53,244 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:53,244 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:B 2024-11-16T05:41:53,244 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:A 2024-11-16T05:41:53,245 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:53,245 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:53,245 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/C is initiating minor compaction (all files) 2024-11-16T05:41:53,245 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/C in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:53,245 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/89e1312305314003a40f4a9f35c00cf7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a675398464cc4767935e54cd382a61eb, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/32b7cac3111245ebb87b826395bee70d] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=33.7 K 2024-11-16T05:41:53,245 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e1312305314003a40f4a9f35c00cf7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731735709736 2024-11-16T05:41:53,246 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a675398464cc4767935e54cd382a61eb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731735710916 2024-11-16T05:41:53,246 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32b7cac3111245ebb87b826395bee70d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731735711606 2024-11-16T05:41:53,252 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#C#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:53,252 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/d860764752cd4757b0f834d492ad6f2f is 50, key is test_row_0/C:col10/1731735711609/Put/seqid=0 2024-11-16T05:41:53,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742339_1515 (size=12731) 2024-11-16T05:41:53,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735773258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735773268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735773268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,276 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d12aeb06061e40a7ae5c3b8874b86c4f 2024-11-16T05:41:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/811a3895a2d74ee49e7806b82c855da3 is 50, key is test_row_0/B:col10/1731735711961/Put/seqid=0 2024-11-16T05:41:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735773278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742340_1516 (size=9757) 2024-11-16T05:41:53,285 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/811a3895a2d74ee49e7806b82c855da3 2024-11-16T05:41:53,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/4e026bde20b64abd89ab3b769818fcca is 50, key is test_row_0/C:col10/1731735711961/Put/seqid=0 2024-11-16T05:41:53,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742341_1517 (size=9757) 2024-11-16T05:41:53,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735773468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735773478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735773478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735773488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,662 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/d860764752cd4757b0f834d492ad6f2f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d860764752cd4757b0f834d492ad6f2f 2024-11-16T05:41:53,666 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/C of df3e8cf5ba630b1a9dfd75e69902d58f into d860764752cd4757b0f834d492ad6f2f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:53,666 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:53,666 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/C, priority=13, startTime=1731735712800; duration=0sec 2024-11-16T05:41:53,666 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:53,666 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:C 2024-11-16T05:41:53,694 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/4e026bde20b64abd89ab3b769818fcca 2024-11-16T05:41:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/d12aeb06061e40a7ae5c3b8874b86c4f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d12aeb06061e40a7ae5c3b8874b86c4f 2024-11-16T05:41:53,700 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d12aeb06061e40a7ae5c3b8874b86c4f, entries=100, sequenceid=256, filesize=9.5 K 2024-11-16T05:41:53,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/811a3895a2d74ee49e7806b82c855da3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/811a3895a2d74ee49e7806b82c855da3 2024-11-16T05:41:53,703 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/811a3895a2d74ee49e7806b82c855da3, entries=100, sequenceid=256, filesize=9.5 K 2024-11-16T05:41:53,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/4e026bde20b64abd89ab3b769818fcca as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/4e026bde20b64abd89ab3b769818fcca 2024-11-16T05:41:53,706 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/4e026bde20b64abd89ab3b769818fcca, entries=100, sequenceid=256, filesize=9.5 K 2024-11-16T05:41:53,706 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for df3e8cf5ba630b1a9dfd75e69902d58f in 838ms, sequenceid=256, compaction requested=false 2024-11-16T05:41:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-16T05:41:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-16T05:41:53,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-16T05:41:53,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7560 sec 2024-11-16T05:41:53,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.7610 sec 2024-11-16T05:41:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:53,783 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:53,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T05:41:53,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/9e626fdefa1d44d8b7492add264ae943 is 50, key is test_row_0/A:col10/1731735713149/Put/seqid=0 2024-11-16T05:41:53,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742342_1518 (size=14741) 2024-11-16T05:41:53,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735773795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735773796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735773798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735773799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735773908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735773908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:53,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735773908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T05:41:54,098 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:54,099 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-16T05:41:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T05:41:54,100 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:54,101 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:54,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:54,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735774118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735774119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:54,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735774128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:54,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/9e626fdefa1d44d8b7492add264ae943 2024-11-16T05:41:54,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/95b726a6592340cca0816671197207d4 is 50, key is test_row_0/B:col10/1731735713149/Put/seqid=0 2024-11-16T05:41:54,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742343_1519 (size=12301) 2024-11-16T05:41:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T05:41:54,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T05:41:54,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:54,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:54,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:54,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735774317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:54,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T05:41:54,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:54,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:54,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:54,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T05:41:54,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735774428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:54,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735774428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735774448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:54,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T05:41:54,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:54,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:54,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:54,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/95b726a6592340cca0816671197207d4 2024-11-16T05:41:54,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/8e6bd4cdff424c7ea7d412c6dc15122b is 50, key is test_row_0/C:col10/1731735713149/Put/seqid=0 2024-11-16T05:41:54,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742344_1520 (size=12301) 2024-11-16T05:41:54,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T05:41:54,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:54,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:54,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:54,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T05:41:54,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T05:41:54,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:54,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:54,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:54,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:54,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735774948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:54,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735774948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:54,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735774968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:55,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/8e6bd4cdff424c7ea7d412c6dc15122b 2024-11-16T05:41:55,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/9e626fdefa1d44d8b7492add264ae943 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9e626fdefa1d44d8b7492add264ae943 2024-11-16T05:41:55,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T05:41:55,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:55,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:55,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:55,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:55,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:55,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9e626fdefa1d44d8b7492add264ae943, entries=200, sequenceid=287, filesize=14.4 K 2024-11-16T05:41:55,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/95b726a6592340cca0816671197207d4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/95b726a6592340cca0816671197207d4 2024-11-16T05:41:55,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/95b726a6592340cca0816671197207d4, entries=150, sequenceid=287, filesize=12.0 K 2024-11-16T05:41:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/8e6bd4cdff424c7ea7d412c6dc15122b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8e6bd4cdff424c7ea7d412c6dc15122b 2024-11-16T05:41:55,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8e6bd4cdff424c7ea7d412c6dc15122b, entries=150, sequenceid=287, filesize=12.0 K 2024-11-16T05:41:55,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for df3e8cf5ba630b1a9dfd75e69902d58f in 1241ms, sequenceid=287, compaction requested=true 2024-11-16T05:41:55,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:55,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:55,024 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:55,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:55,024 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:55,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:55,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:55,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:55,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:55,025 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:55,025 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:55,025 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/B is initiating minor compaction (all files) 2024-11-16T05:41:55,025 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/A is initiating minor compaction (all files) 2024-11-16T05:41:55,025 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/A in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:55,025 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/B in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:55,025 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/f8478beaaa5e4ccc8d267db90d4d6e2f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d12aeb06061e40a7ae5c3b8874b86c4f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9e626fdefa1d44d8b7492add264ae943] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=36.4 K 2024-11-16T05:41:55,025 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/419a6c777d9a4454a662e99b39cab80c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/811a3895a2d74ee49e7806b82c855da3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/95b726a6592340cca0816671197207d4] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=34.0 K 2024-11-16T05:41:55,026 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8478beaaa5e4ccc8d267db90d4d6e2f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731735711606 2024-11-16T05:41:55,026 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 419a6c777d9a4454a662e99b39cab80c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731735711606 2024-11-16T05:41:55,026 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d12aeb06061e40a7ae5c3b8874b86c4f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731735711961 2024-11-16T05:41:55,026 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 811a3895a2d74ee49e7806b82c855da3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731735711961 2024-11-16T05:41:55,026 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 95b726a6592340cca0816671197207d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731735713149 2024-11-16T05:41:55,026 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e626fdefa1d44d8b7492add264ae943, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731735713140 2024-11-16T05:41:55,032 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#B#compaction#443 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:55,032 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/fcc2eafc99694959b4ec0f799312b31a is 50, key is test_row_0/B:col10/1731735713149/Put/seqid=0 2024-11-16T05:41:55,032 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#A#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:55,033 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/f168a46bbb4f4bd4b35dcdd165607ea8 is 50, key is test_row_0/A:col10/1731735713149/Put/seqid=0 2024-11-16T05:41:55,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742345_1521 (size=12983) 2024-11-16T05:41:55,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742346_1522 (size=12983) 2024-11-16T05:41:55,043 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/f168a46bbb4f4bd4b35dcdd165607ea8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/f168a46bbb4f4bd4b35dcdd165607ea8 2024-11-16T05:41:55,046 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/A of df3e8cf5ba630b1a9dfd75e69902d58f into f168a46bbb4f4bd4b35dcdd165607ea8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:55,046 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:55,046 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/A, priority=13, startTime=1731735715024; duration=0sec 2024-11-16T05:41:55,047 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:55,047 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:A 2024-11-16T05:41:55,047 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:55,048 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:55,048 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/C is initiating minor compaction (all files) 2024-11-16T05:41:55,048 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/C in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:55,048 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d860764752cd4757b0f834d492ad6f2f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/4e026bde20b64abd89ab3b769818fcca, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8e6bd4cdff424c7ea7d412c6dc15122b] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=34.0 K 2024-11-16T05:41:55,049 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d860764752cd4757b0f834d492ad6f2f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731735711606 2024-11-16T05:41:55,049 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e026bde20b64abd89ab3b769818fcca, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731735711961 2024-11-16T05:41:55,049 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e6bd4cdff424c7ea7d412c6dc15122b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731735713149 2024-11-16T05:41:55,062 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:55,063 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/f3b6b8d96a804d3491127b8eedf0a74b is 50, key is test_row_0/C:col10/1731735713149/Put/seqid=0 2024-11-16T05:41:55,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742347_1523 (size=12983) 2024-11-16T05:41:55,078 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/f3b6b8d96a804d3491127b8eedf0a74b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/f3b6b8d96a804d3491127b8eedf0a74b 2024-11-16T05:41:55,082 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/C of df3e8cf5ba630b1a9dfd75e69902d58f into f3b6b8d96a804d3491127b8eedf0a74b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:55,083 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:55,083 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/C, priority=13, startTime=1731735715025; duration=0sec 2024-11-16T05:41:55,083 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:55,083 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:C 2024-11-16T05:41:55,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T05:41:55,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:55,168 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T05:41:55,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/555ad7f438a94506b994b8f5e72ef906 is 50, key is test_row_0/A:col10/1731735713798/Put/seqid=0 2024-11-16T05:41:55,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742348_1524 (size=9857) 2024-11-16T05:41:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T05:41:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:55,445 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/fcc2eafc99694959b4ec0f799312b31a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fcc2eafc99694959b4ec0f799312b31a 2024-11-16T05:41:55,454 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/B of df3e8cf5ba630b1a9dfd75e69902d58f into fcc2eafc99694959b4ec0f799312b31a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:55,454 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:55,454 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/B, priority=13, startTime=1731735715024; duration=0sec 2024-11-16T05:41:55,455 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:55,455 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:B 2024-11-16T05:41:55,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735775475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:55,575 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/555ad7f438a94506b994b8f5e72ef906 2024-11-16T05:41:55,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/665d48ff941640459a1517086ffefc28 is 50, key is test_row_0/B:col10/1731735713798/Put/seqid=0 2024-11-16T05:41:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735775588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742349_1525 (size=9857) 2024-11-16T05:41:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735775798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:55,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735775968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:55,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735775968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:55,973 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_2' in df3e8cf5ba630b1a9dfd75e69902d58f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:55,973 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in df3e8cf5ba630b1a9dfd75e69902d58f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:55,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735775978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:55,982 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_0' in df3e8cf5ba630b1a9dfd75e69902d58f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:41:55,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:55,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:55,990 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/665d48ff941640459a1517086ffefc28 2024-11-16T05:41:55,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/9f6a2fa3499742ca9cc42caa343c7652 is 50, key is test_row_0/C:col10/1731735713798/Put/seqid=0 2024-11-16T05:41:56,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742350_1526 (size=9857) 2024-11-16T05:41:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735776108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T05:41:56,400 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/9f6a2fa3499742ca9cc42caa343c7652 2024-11-16T05:41:56,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/555ad7f438a94506b994b8f5e72ef906 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/555ad7f438a94506b994b8f5e72ef906 2024-11-16T05:41:56,406 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/555ad7f438a94506b994b8f5e72ef906, entries=100, sequenceid=297, filesize=9.6 K 2024-11-16T05:41:56,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/665d48ff941640459a1517086ffefc28 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/665d48ff941640459a1517086ffefc28 2024-11-16T05:41:56,409 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/665d48ff941640459a1517086ffefc28, entries=100, sequenceid=297, filesize=9.6 K 2024-11-16T05:41:56,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/9f6a2fa3499742ca9cc42caa343c7652 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/9f6a2fa3499742ca9cc42caa343c7652 2024-11-16T05:41:56,412 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/9f6a2fa3499742ca9cc42caa343c7652, entries=100, sequenceid=297, filesize=9.6 K 2024-11-16T05:41:56,413 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for df3e8cf5ba630b1a9dfd75e69902d58f in 1245ms, sequenceid=297, compaction requested=false 2024-11-16T05:41:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-16T05:41:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-16T05:41:56,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-16T05:41:56,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3130 sec 2024-11-16T05:41:56,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.3160 sec 2024-11-16T05:41:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:56,623 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:56,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-16T05:41:56,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/cd662d4697b24df49995e1880398b016 is 50, key is test_row_0/A:col10/1731735716622/Put/seqid=0 2024-11-16T05:41:56,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742351_1527 (size=12301) 2024-11-16T05:41:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735776650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:56,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735776758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:56,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735776968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:56,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:56,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:56,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:57,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/cd662d4697b24df49995e1880398b016 2024-11-16T05:41:57,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/4997035110634078947a1503e0099626 is 50, key is test_row_0/B:col10/1731735716622/Put/seqid=0 2024-11-16T05:41:57,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742352_1528 (size=12301) 2024-11-16T05:41:57,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735777278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:57,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/4997035110634078947a1503e0099626 2024-11-16T05:41:57,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/e136a2a2880744c69dff28024ee66b04 is 50, key is test_row_0/C:col10/1731735716622/Put/seqid=0 2024-11-16T05:41:57,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742353_1529 (size=12301) 2024-11-16T05:41:57,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44926 deadline: 1731735777788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:57,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/e136a2a2880744c69dff28024ee66b04 2024-11-16T05:41:57,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/cd662d4697b24df49995e1880398b016 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/cd662d4697b24df49995e1880398b016 2024-11-16T05:41:57,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/cd662d4697b24df49995e1880398b016, entries=150, sequenceid=327, filesize=12.0 K 2024-11-16T05:41:57,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/4997035110634078947a1503e0099626 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/4997035110634078947a1503e0099626 2024-11-16T05:41:57,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/4997035110634078947a1503e0099626, entries=150, sequenceid=327, filesize=12.0 K 2024-11-16T05:41:57,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/e136a2a2880744c69dff28024ee66b04 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/e136a2a2880744c69dff28024ee66b04 2024-11-16T05:41:57,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/e136a2a2880744c69dff28024ee66b04, entries=150, sequenceid=327, filesize=12.0 K 2024-11-16T05:41:57,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for df3e8cf5ba630b1a9dfd75e69902d58f in 1286ms, sequenceid=327, compaction requested=true 2024-11-16T05:41:57,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:57,910 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:57,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:41:57,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:57,910 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:57,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:41:57,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:57,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df3e8cf5ba630b1a9dfd75e69902d58f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:41:57,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:57,911 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:57,911 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/A is initiating minor compaction (all files) 2024-11-16T05:41:57,911 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:57,911 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/A in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:57,911 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/B is initiating minor compaction (all files) 2024-11-16T05:41:57,911 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/B in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:57,911 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/f168a46bbb4f4bd4b35dcdd165607ea8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/555ad7f438a94506b994b8f5e72ef906, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/cd662d4697b24df49995e1880398b016] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=34.3 K 2024-11-16T05:41:57,911 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fcc2eafc99694959b4ec0f799312b31a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/665d48ff941640459a1517086ffefc28, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/4997035110634078947a1503e0099626] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=34.3 K 2024-11-16T05:41:57,912 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f168a46bbb4f4bd4b35dcdd165607ea8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731735713149 2024-11-16T05:41:57,912 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc2eafc99694959b4ec0f799312b31a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731735713149 2024-11-16T05:41:57,912 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 665d48ff941640459a1517086ffefc28, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731735713795 2024-11-16T05:41:57,912 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 555ad7f438a94506b994b8f5e72ef906, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731735713795 2024-11-16T05:41:57,912 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4997035110634078947a1503e0099626, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731735715458 2024-11-16T05:41:57,912 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd662d4697b24df49995e1880398b016, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731735715458 2024-11-16T05:41:57,919 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#A#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:57,919 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/574a021fe3c24e6aa1b6843f09f8a445 is 50, key is test_row_0/A:col10/1731735716622/Put/seqid=0 2024-11-16T05:41:57,927 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#B#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:57,927 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/65ea09af0f364a69b14a08fd3963e5ec is 50, key is test_row_0/B:col10/1731735716622/Put/seqid=0 2024-11-16T05:41:57,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742354_1530 (size=13085) 2024-11-16T05:41:57,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742355_1531 (size=13085) 2024-11-16T05:41:57,948 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/65ea09af0f364a69b14a08fd3963e5ec as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/65ea09af0f364a69b14a08fd3963e5ec 2024-11-16T05:41:57,952 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/B of df3e8cf5ba630b1a9dfd75e69902d58f into 65ea09af0f364a69b14a08fd3963e5ec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:57,952 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:57,952 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/B, priority=13, startTime=1731735717910; duration=0sec 2024-11-16T05:41:57,953 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:41:57,953 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:B 2024-11-16T05:41:57,953 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:41:57,953 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:41:57,954 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): df3e8cf5ba630b1a9dfd75e69902d58f/C is initiating minor compaction (all files) 2024-11-16T05:41:57,954 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df3e8cf5ba630b1a9dfd75e69902d58f/C in TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:57,954 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/f3b6b8d96a804d3491127b8eedf0a74b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/9f6a2fa3499742ca9cc42caa343c7652, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/e136a2a2880744c69dff28024ee66b04] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp, totalSize=34.3 K 2024-11-16T05:41:57,954 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f3b6b8d96a804d3491127b8eedf0a74b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731735713149 2024-11-16T05:41:57,954 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f6a2fa3499742ca9cc42caa343c7652, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731735713795 2024-11-16T05:41:57,954 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e136a2a2880744c69dff28024ee66b04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731735715458 2024-11-16T05:41:57,960 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df3e8cf5ba630b1a9dfd75e69902d58f#C#compaction#454 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:41:57,961 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/b0f95d6335bb4ea8af29f362533f58d1 is 50, key is test_row_0/C:col10/1731735716622/Put/seqid=0 2024-11-16T05:41:57,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742356_1532 (size=13085) 2024-11-16T05:41:57,977 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/b0f95d6335bb4ea8af29f362533f58d1 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/b0f95d6335bb4ea8af29f362533f58d1 2024-11-16T05:41:57,981 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/C of df3e8cf5ba630b1a9dfd75e69902d58f into b0f95d6335bb4ea8af29f362533f58d1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:57,981 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:57,981 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/C, priority=13, startTime=1731735717910; duration=0sec 2024-11-16T05:41:57,981 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:57,981 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:C 2024-11-16T05:41:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:41:57,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:41:58,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/60c8812257834818a9ae998a93478b34 is 50, key is test_row_0/A:col10/1731735717980/Put/seqid=0 2024-11-16T05:41:58,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742357_1533 (size=14741) 2024-11-16T05:41:58,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/60c8812257834818a9ae998a93478b34 2024-11-16T05:41:58,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/8b050704d6254d76beb1a2ba89d2982d is 50, key is test_row_0/B:col10/1731735717980/Put/seqid=0 2024-11-16T05:41:58,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742358_1534 (size=12301) 2024-11-16T05:41:58,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735778067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735778068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:58,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:58,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735778072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:58,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:58,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735778188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:58,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735778188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:58,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735778188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T05:41:58,268 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:41:58,269 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:41:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-16T05:41:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T05:41:58,270 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:41:58,270 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:41:58,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:41:58,335 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/574a021fe3c24e6aa1b6843f09f8a445 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/574a021fe3c24e6aa1b6843f09f8a445 2024-11-16T05:41:58,338 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df3e8cf5ba630b1a9dfd75e69902d58f/A of df3e8cf5ba630b1a9dfd75e69902d58f into 574a021fe3c24e6aa1b6843f09f8a445(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:41:58,339 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:58,339 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., storeName=df3e8cf5ba630b1a9dfd75e69902d58f/A, priority=13, startTime=1731735717910; duration=0sec 2024-11-16T05:41:58,339 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:41:58,339 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df3e8cf5ba630b1a9dfd75e69902d58f:A 2024-11-16T05:41:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T05:41:58,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44942 deadline: 1731735778408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:58,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44972 deadline: 1731735778408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:41:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44958 deadline: 1731735778409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:41:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 , the old value is region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:41:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., hostname=3456ee6a3164,35555,1731735575219, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:41:58,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-16T05:41:58,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:58,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:58,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:58,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:41:58,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/8b050704d6254d76beb1a2ba89d2982d 2024-11-16T05:41:58,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/815fd004d3364ea89487cbcba2d75843 is 50, key is test_row_0/C:col10/1731735717980/Put/seqid=0 2024-11-16T05:41:58,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742359_1535 (size=12301) 2024-11-16T05:41:58,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/815fd004d3364ea89487cbcba2d75843 2024-11-16T05:41:58,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/60c8812257834818a9ae998a93478b34 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/60c8812257834818a9ae998a93478b34 2024-11-16T05:41:58,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/60c8812257834818a9ae998a93478b34, entries=200, sequenceid=340, filesize=14.4 K 2024-11-16T05:41:58,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/8b050704d6254d76beb1a2ba89d2982d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/8b050704d6254d76beb1a2ba89d2982d 2024-11-16T05:41:58,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/8b050704d6254d76beb1a2ba89d2982d, entries=150, sequenceid=340, filesize=12.0 K 2024-11-16T05:41:58,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/815fd004d3364ea89487cbcba2d75843 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/815fd004d3364ea89487cbcba2d75843 2024-11-16T05:41:58,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/815fd004d3364ea89487cbcba2d75843, entries=150, sequenceid=340, filesize=12.0 K 2024-11-16T05:41:58,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for df3e8cf5ba630b1a9dfd75e69902d58f in 466ms, sequenceid=340, compaction requested=false 2024-11-16T05:41:58,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:58,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-16T05:41:58,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:58,576 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T05:41:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/b21925becf164b30816ff4b593bb0cc9 is 50, key is test_row_0/A:col10/1731735718058/Put/seqid=0 2024-11-16T05:41:58,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742360_1536 (size=12301) 2024-11-16T05:41:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T05:41:58,683 INFO [Thread-2063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2063. 2024-11-16T05:41:58,683 DEBUG [Thread-2063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:58,683 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,683 INFO [Thread-2061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2061. 2024-11-16T05:41:58,683 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,683 DEBUG [Thread-2061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:58,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:58,683 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,683 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:58,685 INFO [Thread-2065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2065. 2024-11-16T05:41:58,685 DEBUG [Thread-2065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:58,685 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,685 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:58,688 INFO [Thread-2067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2067. 2024-11-16T05:41:58,688 INFO [Thread-2069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2069. 2024-11-16T05:41:58,688 DEBUG [Thread-2067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:58,688 DEBUG [Thread-2069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:58,688 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,688 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,688 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,688 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:58,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:41:58,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. as already flushing 2024-11-16T05:41:58,720 INFO [Thread-2056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2056. 2024-11-16T05:41:58,720 DEBUG [Thread-2056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:58,720 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,720 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,720 INFO [Thread-2058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2058. 2024-11-16T05:41:58,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:58,720 DEBUG [Thread-2058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:58,720 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,721 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:58,730 INFO [Thread-2054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2054. 2024-11-16T05:41:58,730 DEBUG [Thread-2054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:58,730 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,730 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:58,813 INFO [Thread-2052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2052. 2024-11-16T05:41:58,813 DEBUG [Thread-2052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:41:58,813 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,814 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:41:58,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:41:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T05:41:58,983 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/b21925becf164b30816ff4b593bb0cc9 2024-11-16T05:41:58,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/bdc6d99897f44b03b9133098b84ef935 is 50, key is test_row_0/B:col10/1731735718058/Put/seqid=0 2024-11-16T05:41:58,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742361_1537 (size=12301) 2024-11-16T05:41:59,401 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/bdc6d99897f44b03b9133098b84ef935 2024-11-16T05:41:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T05:41:59,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/554a74aac681437ab0bb2a80a17c02ac is 50, key is test_row_0/C:col10/1731735718058/Put/seqid=0 2024-11-16T05:41:59,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742362_1538 (size=12301) 2024-11-16T05:41:59,822 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/554a74aac681437ab0bb2a80a17c02ac 2024-11-16T05:41:59,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/b21925becf164b30816ff4b593bb0cc9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/b21925becf164b30816ff4b593bb0cc9 2024-11-16T05:41:59,837 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/b21925becf164b30816ff4b593bb0cc9, entries=150, sequenceid=366, filesize=12.0 K 2024-11-16T05:41:59,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/bdc6d99897f44b03b9133098b84ef935 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/bdc6d99897f44b03b9133098b84ef935 2024-11-16T05:41:59,840 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/bdc6d99897f44b03b9133098b84ef935, entries=150, sequenceid=366, filesize=12.0 K 2024-11-16T05:41:59,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/554a74aac681437ab0bb2a80a17c02ac as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/554a74aac681437ab0bb2a80a17c02ac 2024-11-16T05:41:59,844 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/554a74aac681437ab0bb2a80a17c02ac, entries=150, sequenceid=366, filesize=12.0 K 2024-11-16T05:41:59,845 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for df3e8cf5ba630b1a9dfd75e69902d58f in 1269ms, sequenceid=366, compaction requested=true 2024-11-16T05:41:59,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for df3e8cf5ba630b1a9dfd75e69902d58f: 2024-11-16T05:41:59,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:41:59,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-16T05:41:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-16T05:41:59,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-16T05:41:59,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5760 sec 2024-11-16T05:41:59,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.5790 sec 2024-11-16T05:42:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T05:42:00,419 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:02,280 INFO [Thread-2050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2050. 2024-11-16T05:42:02,280 DEBUG [Thread-2050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:02,280 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:02,280 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:02,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:02,280 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:42:02,280 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-16T05:42:02,280 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-16T05:42:02,280 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-16T05:42:02,280 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-16T05:42:02,280 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-16T05:42:02,280 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:42:02,280 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:42:02,280 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1599 2024-11-16T05:42:02,280 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4794 rows 2024-11-16T05:42:02,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1612 2024-11-16T05:42:02,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4829 rows 2024-11-16T05:42:02,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1596 2024-11-16T05:42:02,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4788 rows 2024-11-16T05:42:02,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1599 2024-11-16T05:42:02,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4795 rows 2024-11-16T05:42:02,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1602 2024-11-16T05:42:02,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4805 rows 2024-11-16T05:42:02,281 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:42:02,281 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:42:02,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:02,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:02,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:02,282 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:42:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T05:42:02,285 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735722285"}]},"ts":"1731735722285"} 2024-11-16T05:42:02,286 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:42:02,286 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:42:02,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:42:02,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df3e8cf5ba630b1a9dfd75e69902d58f, UNASSIGN}] 2024-11-16T05:42:02,290 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df3e8cf5ba630b1a9dfd75e69902d58f, UNASSIGN 2024-11-16T05:42:02,291 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=df3e8cf5ba630b1a9dfd75e69902d58f, regionState=CLOSING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:02,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df3e8cf5ba630b1a9dfd75e69902d58f, UNASSIGN because future has completed 2024-11-16T05:42:02,292 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:42:02,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:42:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T05:42:02,445 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:42:02,445 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:42:02,445 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing df3e8cf5ba630b1a9dfd75e69902d58f, disabling compactions & flushes 2024-11-16T05:42:02,446 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:42:02,446 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:42:02,446 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. after waiting 0 ms 2024-11-16T05:42:02,446 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:42:02,446 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing df3e8cf5ba630b1a9dfd75e69902d58f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T05:42:02,449 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/a419e054258c4ea3a2a25cff3525ce6d is 50, key is test_row_0/A:col10/1731735722279/Put/seqid=0 2024-11-16T05:42:02,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742363_1539 (size=12301) 2024-11-16T05:42:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T05:42:02,853 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/a419e054258c4ea3a2a25cff3525ce6d 2024-11-16T05:42:02,860 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/a242ab37b3c149ef9f3713d2358223ec is 50, key is test_row_0/B:col10/1731735722279/Put/seqid=0 2024-11-16T05:42:02,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742364_1540 (size=12301) 2024-11-16T05:42:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T05:42:03,264 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/a242ab37b3c149ef9f3713d2358223ec 2024-11-16T05:42:03,270 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/313ae0da20eb4c4aa538bddb7389c36f is 50, key is test_row_0/C:col10/1731735722279/Put/seqid=0 2024-11-16T05:42:03,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742365_1541 (size=12301) 2024-11-16T05:42:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T05:42:03,556 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:42:03,674 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/313ae0da20eb4c4aa538bddb7389c36f 2024-11-16T05:42:03,678 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/A/a419e054258c4ea3a2a25cff3525ce6d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/a419e054258c4ea3a2a25cff3525ce6d 2024-11-16T05:42:03,680 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/a419e054258c4ea3a2a25cff3525ce6d, entries=150, sequenceid=374, filesize=12.0 K 2024-11-16T05:42:03,681 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/B/a242ab37b3c149ef9f3713d2358223ec as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/a242ab37b3c149ef9f3713d2358223ec 2024-11-16T05:42:03,683 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/a242ab37b3c149ef9f3713d2358223ec, entries=150, sequenceid=374, filesize=12.0 K 2024-11-16T05:42:03,683 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/.tmp/C/313ae0da20eb4c4aa538bddb7389c36f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/313ae0da20eb4c4aa538bddb7389c36f 2024-11-16T05:42:03,686 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/313ae0da20eb4c4aa538bddb7389c36f, entries=150, sequenceid=374, filesize=12.0 K 2024-11-16T05:42:03,686 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for df3e8cf5ba630b1a9dfd75e69902d58f in 1240ms, sequenceid=374, compaction requested=true 2024-11-16T05:42:03,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/0b6ace7d1ee240efb05cfa1442dffc54, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9c5b9774bda44120b12a0f4437784bed, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/5eb4ba1f14f94bbeb63e16d4172d05dd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/76cf373a559c40e390a7aacf7f8dcb35, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/b7075cf7eb724ef7b15b0007d1d563f7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d5731d307bf3445fad22b36e585fa60a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/2b0d578c20b74e4eaa1ad74e70e4812b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/c79134bcff914b8290708634bb18058a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/abee94c1d2b24b91b50408f7c46ed296, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/bc201d0e6e654435b71b2e9f4df0370f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/775b9392e0d243a2aac0f5fc0832ea57, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/7714fa5c042943188da10f379f04915c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/83b91edd9b404480b9bc730fe0d570c4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d54147aab5d64a68b0a4c1f62293d8c3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/dde3efc5fe3a4fb88a70a731057f451f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d41536e070444b75a9b88bf7fdb8eb63, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/ab0606bdd1fc494daa9373b2b442bdbc, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/f8478beaaa5e4ccc8d267db90d4d6e2f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d12aeb06061e40a7ae5c3b8874b86c4f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9e626fdefa1d44d8b7492add264ae943, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/f168a46bbb4f4bd4b35dcdd165607ea8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/555ad7f438a94506b994b8f5e72ef906, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/cd662d4697b24df49995e1880398b016] to archive 2024-11-16T05:42:03,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:42:03,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/0b6ace7d1ee240efb05cfa1442dffc54 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/0b6ace7d1ee240efb05cfa1442dffc54 2024-11-16T05:42:03,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9c5b9774bda44120b12a0f4437784bed to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9c5b9774bda44120b12a0f4437784bed 2024-11-16T05:42:03,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/5eb4ba1f14f94bbeb63e16d4172d05dd to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/5eb4ba1f14f94bbeb63e16d4172d05dd 2024-11-16T05:42:03,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/76cf373a559c40e390a7aacf7f8dcb35 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/76cf373a559c40e390a7aacf7f8dcb35 2024-11-16T05:42:03,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/b7075cf7eb724ef7b15b0007d1d563f7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/b7075cf7eb724ef7b15b0007d1d563f7 2024-11-16T05:42:03,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d5731d307bf3445fad22b36e585fa60a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d5731d307bf3445fad22b36e585fa60a 2024-11-16T05:42:03,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/2b0d578c20b74e4eaa1ad74e70e4812b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/2b0d578c20b74e4eaa1ad74e70e4812b 2024-11-16T05:42:03,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/c79134bcff914b8290708634bb18058a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/c79134bcff914b8290708634bb18058a 2024-11-16T05:42:03,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/abee94c1d2b24b91b50408f7c46ed296 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/abee94c1d2b24b91b50408f7c46ed296 2024-11-16T05:42:03,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/bc201d0e6e654435b71b2e9f4df0370f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/bc201d0e6e654435b71b2e9f4df0370f 2024-11-16T05:42:03,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/775b9392e0d243a2aac0f5fc0832ea57 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/775b9392e0d243a2aac0f5fc0832ea57 2024-11-16T05:42:03,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/7714fa5c042943188da10f379f04915c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/7714fa5c042943188da10f379f04915c 2024-11-16T05:42:03,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/83b91edd9b404480b9bc730fe0d570c4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/83b91edd9b404480b9bc730fe0d570c4 2024-11-16T05:42:03,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d54147aab5d64a68b0a4c1f62293d8c3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d54147aab5d64a68b0a4c1f62293d8c3 2024-11-16T05:42:03,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/dde3efc5fe3a4fb88a70a731057f451f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/dde3efc5fe3a4fb88a70a731057f451f 2024-11-16T05:42:03,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d41536e070444b75a9b88bf7fdb8eb63 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d41536e070444b75a9b88bf7fdb8eb63 2024-11-16T05:42:03,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/ab0606bdd1fc494daa9373b2b442bdbc to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/ab0606bdd1fc494daa9373b2b442bdbc 2024-11-16T05:42:03,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/f8478beaaa5e4ccc8d267db90d4d6e2f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/f8478beaaa5e4ccc8d267db90d4d6e2f 2024-11-16T05:42:03,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d12aeb06061e40a7ae5c3b8874b86c4f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/d12aeb06061e40a7ae5c3b8874b86c4f 2024-11-16T05:42:03,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9e626fdefa1d44d8b7492add264ae943 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/9e626fdefa1d44d8b7492add264ae943 2024-11-16T05:42:03,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/f168a46bbb4f4bd4b35dcdd165607ea8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/f168a46bbb4f4bd4b35dcdd165607ea8 2024-11-16T05:42:03,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/555ad7f438a94506b994b8f5e72ef906 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/555ad7f438a94506b994b8f5e72ef906 2024-11-16T05:42:03,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/cd662d4697b24df49995e1880398b016 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/cd662d4697b24df49995e1880398b016 2024-11-16T05:42:03,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/85782b287c454bd7a181c9a0a730b827, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/e3fc23a83b5a405c9e73036d7d93afe7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/2540807052a54efda4366da704d9e38f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/8e6cf6af52cc491487d654b686d0d6b1, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/61ed7d807ad142758922749c7715bad8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/ef20351037ee478f948e48f2ec6d2d25, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/f7fdde764a65404a98072a46b8bf7ba2, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/c834f64135874bddbc14c4c50c5319ed, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/667b96969f544459a39bf05e31fd51ec, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/70bb8d9d91d54bf48745fcfd3dba571b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/9adb1992658247e597b37b118eccea14, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/0f3a9804f8a242828ccc31ce01a5b29e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/6c44b3b941f94918b3ad4584f45ce6c7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/75d226c0009141acafe3e97a45d78339, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/3a68993d158e47bc9c208bfce0328b35, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fb7714ce4d564a8bbe45a97947a537c7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/419a6c777d9a4454a662e99b39cab80c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/1d8a3136c0b94ce3a25f874e5487ea61, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/811a3895a2d74ee49e7806b82c855da3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fcc2eafc99694959b4ec0f799312b31a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/95b726a6592340cca0816671197207d4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/665d48ff941640459a1517086ffefc28, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/4997035110634078947a1503e0099626] to archive 2024-11-16T05:42:03,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:42:03,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/85782b287c454bd7a181c9a0a730b827 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/85782b287c454bd7a181c9a0a730b827 2024-11-16T05:42:03,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/e3fc23a83b5a405c9e73036d7d93afe7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/e3fc23a83b5a405c9e73036d7d93afe7 2024-11-16T05:42:03,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/2540807052a54efda4366da704d9e38f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/2540807052a54efda4366da704d9e38f 2024-11-16T05:42:03,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/8e6cf6af52cc491487d654b686d0d6b1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/8e6cf6af52cc491487d654b686d0d6b1 2024-11-16T05:42:03,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/61ed7d807ad142758922749c7715bad8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/61ed7d807ad142758922749c7715bad8 2024-11-16T05:42:03,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/ef20351037ee478f948e48f2ec6d2d25 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/ef20351037ee478f948e48f2ec6d2d25 2024-11-16T05:42:03,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/f7fdde764a65404a98072a46b8bf7ba2 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/f7fdde764a65404a98072a46b8bf7ba2 2024-11-16T05:42:03,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/c834f64135874bddbc14c4c50c5319ed to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/c834f64135874bddbc14c4c50c5319ed 2024-11-16T05:42:03,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/667b96969f544459a39bf05e31fd51ec to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/667b96969f544459a39bf05e31fd51ec 2024-11-16T05:42:03,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/70bb8d9d91d54bf48745fcfd3dba571b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/70bb8d9d91d54bf48745fcfd3dba571b 2024-11-16T05:42:03,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/9adb1992658247e597b37b118eccea14 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/9adb1992658247e597b37b118eccea14 2024-11-16T05:42:03,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/0f3a9804f8a242828ccc31ce01a5b29e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/0f3a9804f8a242828ccc31ce01a5b29e 2024-11-16T05:42:03,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/6c44b3b941f94918b3ad4584f45ce6c7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/6c44b3b941f94918b3ad4584f45ce6c7 2024-11-16T05:42:03,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/75d226c0009141acafe3e97a45d78339 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/75d226c0009141acafe3e97a45d78339 2024-11-16T05:42:03,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/3a68993d158e47bc9c208bfce0328b35 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/3a68993d158e47bc9c208bfce0328b35 2024-11-16T05:42:03,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fb7714ce4d564a8bbe45a97947a537c7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fb7714ce4d564a8bbe45a97947a537c7 2024-11-16T05:42:03,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/419a6c777d9a4454a662e99b39cab80c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/419a6c777d9a4454a662e99b39cab80c 2024-11-16T05:42:03,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/1d8a3136c0b94ce3a25f874e5487ea61 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/1d8a3136c0b94ce3a25f874e5487ea61 2024-11-16T05:42:03,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/811a3895a2d74ee49e7806b82c855da3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/811a3895a2d74ee49e7806b82c855da3 2024-11-16T05:42:03,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fcc2eafc99694959b4ec0f799312b31a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/fcc2eafc99694959b4ec0f799312b31a 2024-11-16T05:42:03,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/95b726a6592340cca0816671197207d4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/95b726a6592340cca0816671197207d4 2024-11-16T05:42:03,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/665d48ff941640459a1517086ffefc28 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/665d48ff941640459a1517086ffefc28 2024-11-16T05:42:03,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/4997035110634078947a1503e0099626 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/4997035110634078947a1503e0099626 2024-11-16T05:42:03,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d15f92ee08f5452ab1dc2a0b51aa05df, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/7c124dee3987478f8abdc7aa0c34ae6e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6eb4164dba6442f9bbd519fd165f19cc, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/3ce17a6f16244293949aba393bfc2178, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/69991208ad6b4979889c4da6a39d9e32, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a4385621bc6a4897beb885b49ce1ebe0, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/166367fb47bf40b8ac20a32c2f3562ea, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/dd86a1dcceed4796bb8d6f33bbdf6cbd, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/030ab0fd6b574d8ab69b23d2e90978b7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8bdcd575d04a4e69b482aadf53053903, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d5b9ad59c4e74db4bcc8c26bff7d2183, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/860253e544e54e378b08b83972dd7b66, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6e6a886be2f24810b100c5243c00a449, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/89e1312305314003a40f4a9f35c00cf7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/5e9ff20b4bb646fb8d602579728f07d3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a675398464cc4767935e54cd382a61eb, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d860764752cd4757b0f834d492ad6f2f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/32b7cac3111245ebb87b826395bee70d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/4e026bde20b64abd89ab3b769818fcca, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/f3b6b8d96a804d3491127b8eedf0a74b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8e6bd4cdff424c7ea7d412c6dc15122b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/9f6a2fa3499742ca9cc42caa343c7652, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/e136a2a2880744c69dff28024ee66b04] to archive 2024-11-16T05:42:03,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:42:03,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d15f92ee08f5452ab1dc2a0b51aa05df to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d15f92ee08f5452ab1dc2a0b51aa05df 2024-11-16T05:42:03,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/7c124dee3987478f8abdc7aa0c34ae6e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/7c124dee3987478f8abdc7aa0c34ae6e 2024-11-16T05:42:03,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6eb4164dba6442f9bbd519fd165f19cc to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6eb4164dba6442f9bbd519fd165f19cc 2024-11-16T05:42:03,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/3ce17a6f16244293949aba393bfc2178 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/3ce17a6f16244293949aba393bfc2178 2024-11-16T05:42:03,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/69991208ad6b4979889c4da6a39d9e32 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/69991208ad6b4979889c4da6a39d9e32 2024-11-16T05:42:03,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a4385621bc6a4897beb885b49ce1ebe0 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a4385621bc6a4897beb885b49ce1ebe0 2024-11-16T05:42:03,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/166367fb47bf40b8ac20a32c2f3562ea to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/166367fb47bf40b8ac20a32c2f3562ea 2024-11-16T05:42:03,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/dd86a1dcceed4796bb8d6f33bbdf6cbd to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/dd86a1dcceed4796bb8d6f33bbdf6cbd 2024-11-16T05:42:03,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/030ab0fd6b574d8ab69b23d2e90978b7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/030ab0fd6b574d8ab69b23d2e90978b7 2024-11-16T05:42:03,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8bdcd575d04a4e69b482aadf53053903 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8bdcd575d04a4e69b482aadf53053903 2024-11-16T05:42:03,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d5b9ad59c4e74db4bcc8c26bff7d2183 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d5b9ad59c4e74db4bcc8c26bff7d2183 2024-11-16T05:42:03,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/860253e544e54e378b08b83972dd7b66 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/860253e544e54e378b08b83972dd7b66 2024-11-16T05:42:03,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6e6a886be2f24810b100c5243c00a449 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/6e6a886be2f24810b100c5243c00a449 2024-11-16T05:42:03,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/89e1312305314003a40f4a9f35c00cf7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/89e1312305314003a40f4a9f35c00cf7 2024-11-16T05:42:03,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/5e9ff20b4bb646fb8d602579728f07d3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/5e9ff20b4bb646fb8d602579728f07d3 2024-11-16T05:42:03,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a675398464cc4767935e54cd382a61eb to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/a675398464cc4767935e54cd382a61eb 2024-11-16T05:42:03,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d860764752cd4757b0f834d492ad6f2f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/d860764752cd4757b0f834d492ad6f2f 2024-11-16T05:42:03,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/32b7cac3111245ebb87b826395bee70d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/32b7cac3111245ebb87b826395bee70d 2024-11-16T05:42:03,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/4e026bde20b64abd89ab3b769818fcca to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/4e026bde20b64abd89ab3b769818fcca 2024-11-16T05:42:03,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/f3b6b8d96a804d3491127b8eedf0a74b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/f3b6b8d96a804d3491127b8eedf0a74b 2024-11-16T05:42:03,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8e6bd4cdff424c7ea7d412c6dc15122b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/8e6bd4cdff424c7ea7d412c6dc15122b 2024-11-16T05:42:03,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/9f6a2fa3499742ca9cc42caa343c7652 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/9f6a2fa3499742ca9cc42caa343c7652 2024-11-16T05:42:03,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/e136a2a2880744c69dff28024ee66b04 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/e136a2a2880744c69dff28024ee66b04 2024-11-16T05:42:03,767 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/recovered.edits/377.seqid, newMaxSeqId=377, maxSeqId=1 2024-11-16T05:42:03,767 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. 2024-11-16T05:42:03,768 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for df3e8cf5ba630b1a9dfd75e69902d58f: Waiting for close lock at 1731735722445Running coprocessor pre-close hooks at 1731735722445Disabling compacts and flushes for region at 1731735722445Disabling writes for close at 1731735722446 (+1 ms)Obtaining lock to block concurrent updates at 1731735722446Preparing flush snapshotting stores in df3e8cf5ba630b1a9dfd75e69902d58f at 1731735722446Finished memstore snapshotting TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731735722446Flushing stores of TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f. at 1731735722446Flushing df3e8cf5ba630b1a9dfd75e69902d58f/A: creating writer at 1731735722446Flushing df3e8cf5ba630b1a9dfd75e69902d58f/A: appending metadata at 1731735722448 (+2 ms)Flushing df3e8cf5ba630b1a9dfd75e69902d58f/A: closing flushed file at 1731735722448Flushing df3e8cf5ba630b1a9dfd75e69902d58f/B: creating writer at 1731735722857 (+409 ms)Flushing df3e8cf5ba630b1a9dfd75e69902d58f/B: appending metadata at 1731735722860 (+3 ms)Flushing df3e8cf5ba630b1a9dfd75e69902d58f/B: closing flushed file at 1731735722860Flushing df3e8cf5ba630b1a9dfd75e69902d58f/C: creating writer at 1731735723267 (+407 ms)Flushing df3e8cf5ba630b1a9dfd75e69902d58f/C: appending metadata at 1731735723270 (+3 ms)Flushing df3e8cf5ba630b1a9dfd75e69902d58f/C: closing flushed file at 1731735723270Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2295ef97: reopening flushed file at 1731735723677 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58be4f77: reopening flushed file at 1731735723680 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fc9691e: reopening flushed file at 1731735723683 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for df3e8cf5ba630b1a9dfd75e69902d58f in 1240ms, sequenceid=374, compaction requested=true at 1731735723686 (+3 ms)Writing region close event to WAL at 1731735723762 (+76 ms)Running coprocessor post-close hooks at 1731735723767 (+5 ms)Closed at 1731735723767 2024-11-16T05:42:03,770 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:42:03,771 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=df3e8cf5ba630b1a9dfd75e69902d58f, regionState=CLOSED 2024-11-16T05:42:03,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:42:03,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-16T05:42:03,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure df3e8cf5ba630b1a9dfd75e69902d58f, server=3456ee6a3164,35555,1731735575219 in 1.4800 sec 2024-11-16T05:42:03,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-16T05:42:03,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df3e8cf5ba630b1a9dfd75e69902d58f, UNASSIGN in 1.4860 sec 2024-11-16T05:42:03,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-16T05:42:03,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4910 sec 2024-11-16T05:42:03,780 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735723780"}]},"ts":"1731735723780"} 2024-11-16T05:42:03,782 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:42:03,782 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:42:03,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5010 sec 2024-11-16T05:42:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T05:42:04,439 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:04,439 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:42:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:04,441 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-16T05:42:04,442 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:04,444 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:42:04,446 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/recovered.edits] 2024-11-16T05:42:04,448 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/574a021fe3c24e6aa1b6843f09f8a445 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/574a021fe3c24e6aa1b6843f09f8a445 2024-11-16T05:42:04,449 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/60c8812257834818a9ae998a93478b34 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/60c8812257834818a9ae998a93478b34 2024-11-16T05:42:04,454 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/a419e054258c4ea3a2a25cff3525ce6d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/a419e054258c4ea3a2a25cff3525ce6d 2024-11-16T05:42:04,455 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/b21925becf164b30816ff4b593bb0cc9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/A/b21925becf164b30816ff4b593bb0cc9 2024-11-16T05:42:04,457 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/65ea09af0f364a69b14a08fd3963e5ec to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/65ea09af0f364a69b14a08fd3963e5ec 2024-11-16T05:42:04,458 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/8b050704d6254d76beb1a2ba89d2982d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/8b050704d6254d76beb1a2ba89d2982d 2024-11-16T05:42:04,459 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/a242ab37b3c149ef9f3713d2358223ec to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/a242ab37b3c149ef9f3713d2358223ec 2024-11-16T05:42:04,460 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/bdc6d99897f44b03b9133098b84ef935 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/B/bdc6d99897f44b03b9133098b84ef935 2024-11-16T05:42:04,462 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/313ae0da20eb4c4aa538bddb7389c36f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/313ae0da20eb4c4aa538bddb7389c36f 2024-11-16T05:42:04,463 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/554a74aac681437ab0bb2a80a17c02ac to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/554a74aac681437ab0bb2a80a17c02ac 2024-11-16T05:42:04,463 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/815fd004d3364ea89487cbcba2d75843 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/815fd004d3364ea89487cbcba2d75843 2024-11-16T05:42:04,464 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/b0f95d6335bb4ea8af29f362533f58d1 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/C/b0f95d6335bb4ea8af29f362533f58d1 2024-11-16T05:42:04,466 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/recovered.edits/377.seqid to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f/recovered.edits/377.seqid 2024-11-16T05:42:04,467 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/df3e8cf5ba630b1a9dfd75e69902d58f 2024-11-16T05:42:04,467 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:42:04,468 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:04,470 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:42:04,472 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:42:04,473 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:04,473 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:42:04,474 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731735724473"}]},"ts":"9223372036854775807"} 2024-11-16T05:42:04,476 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:42:04,476 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => df3e8cf5ba630b1a9dfd75e69902d58f, NAME => 'TestAcidGuarantees,,1731735697473.df3e8cf5ba630b1a9dfd75e69902d58f.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:42:04,476 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:42:04,476 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731735724476"}]},"ts":"9223372036854775807"} 2024-11-16T05:42:04,477 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:42:04,478 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:04,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-11-16T05:42:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-16T05:42:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:42:04,552 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:04,560 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237 (was 237), OpenFileDescriptor=445 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 374) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4067 (was 3365) - AvailableMemoryMB LEAK? - 2024-11-16T05:42:04,571 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=387, ProcessCount=11, AvailableMemoryMB=4065 2024-11-16T05:42:04,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:42:04,573 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:42:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:04,576 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:42:04,576 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:04,576 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-16T05:42:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:42:04,577 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:42:04,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742366_1542 (size=959) 2024-11-16T05:42:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:42:04,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:42:04,949 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:42:04,983 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d 2024-11-16T05:42:04,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742367_1543 (size=53) 2024-11-16T05:42:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:42:05,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:42:05,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ebfb6a6d32a314c2093cd1145e998024, disabling compactions & flushes 2024-11-16T05:42:05,390 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:05,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:05,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. after waiting 0 ms 2024-11-16T05:42:05,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:05,390 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:05,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ebfb6a6d32a314c2093cd1145e998024: Waiting for close lock at 1731735725390Disabling compacts and flushes for region at 1731735725390Disabling writes for close at 1731735725390Writing region close event to WAL at 1731735725390Closed at 1731735725390 2024-11-16T05:42:05,392 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:42:05,392 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731735725392"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731735725392"}]},"ts":"1731735725392"} 2024-11-16T05:42:05,393 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:42:05,394 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:42:05,395 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735725394"}]},"ts":"1731735725394"} 2024-11-16T05:42:05,396 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:42:05,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, ASSIGN}] 2024-11-16T05:42:05,397 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, ASSIGN 2024-11-16T05:42:05,398 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, ASSIGN; state=OFFLINE, location=3456ee6a3164,35555,1731735575219; forceNewPlan=false, retain=false 2024-11-16T05:42:05,549 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ebfb6a6d32a314c2093cd1145e998024, regionState=OPENING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:05,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, ASSIGN because future has completed 2024-11-16T05:42:05,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:42:05,707 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:05,707 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:42:05,708 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,708 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:42:05,708 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,708 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,710 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,712 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebfb6a6d32a314c2093cd1145e998024 columnFamilyName A 2024-11-16T05:42:05,712 DEBUG [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:05,713 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(327): Store=ebfb6a6d32a314c2093cd1145e998024/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:42:05,713 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,715 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebfb6a6d32a314c2093cd1145e998024 columnFamilyName B 2024-11-16T05:42:05,715 DEBUG [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:05,715 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(327): Store=ebfb6a6d32a314c2093cd1145e998024/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:42:05,716 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,717 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebfb6a6d32a314c2093cd1145e998024 columnFamilyName C 2024-11-16T05:42:05,717 DEBUG [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:05,718 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(327): Store=ebfb6a6d32a314c2093cd1145e998024/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:42:05,718 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:42:05,719 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,719 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,721 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,721 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,721 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:42:05,723 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,725 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:42:05,726 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened ebfb6a6d32a314c2093cd1145e998024; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71308196, jitterRate=0.06257492303848267}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:42:05,726 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:05,726 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for ebfb6a6d32a314c2093cd1145e998024: Running coprocessor pre-open hook at 1731735725708Writing region info on filesystem at 1731735725708Initializing all the Stores at 1731735725709 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735725709Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735725710 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735725710Cleaning up temporary data from old regions at 1731735725721 (+11 ms)Running coprocessor post-open hooks at 1731735725726 (+5 ms)Region opened successfully at 1731735725726 2024-11-16T05:42:05,727 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., pid=150, masterSystemTime=1731735725702 2024-11-16T05:42:05,729 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:05,729 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:05,729 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ebfb6a6d32a314c2093cd1145e998024, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:05,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:42:05,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-16T05:42:05,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 in 181 msec 2024-11-16T05:42:05,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-16T05:42:05,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, ASSIGN in 337 msec 2024-11-16T05:42:05,736 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:42:05,736 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735725736"}]},"ts":"1731735725736"} 2024-11-16T05:42:05,737 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:42:05,738 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:42:05,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-16T05:42:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:42:06,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:06,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cec06c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:06,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:06,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:06,731 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ccf36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:06,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256a7ed0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:06,734 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:06,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:06,737 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:42:06,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,39409,1731735574536 2024-11-16T05:42:06,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e21d5ad 2024-11-16T05:42:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:42:06,739 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:42:06,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:42:06,740 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:42:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:06,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742368_1544 (size=995) 2024-11-16T05:42:07,152 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T05:42:07,152 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T05:42:07,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:42:07,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, REOPEN/MOVE}] 2024-11-16T05:42:07,157 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, REOPEN/MOVE 2024-11-16T05:42:07,157 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=ebfb6a6d32a314c2093cd1145e998024, regionState=CLOSING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,158 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39409 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=3456ee6a3164,35555,1731735575219, table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:42:07,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, REOPEN/MOVE because future has completed 2024-11-16T05:42:07,159 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:42:07,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:42:07,311 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,311 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:42:07,311 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing ebfb6a6d32a314c2093cd1145e998024, disabling compactions & flushes 2024-11-16T05:42:07,311 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:07,311 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:07,311 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. after waiting 0 ms 2024-11-16T05:42:07,311 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:07,314 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:42:07,314 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:07,314 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for ebfb6a6d32a314c2093cd1145e998024: Waiting for close lock at 1731735727311Running coprocessor pre-close hooks at 1731735727311Disabling compacts and flushes for region at 1731735727311Disabling writes for close at 1731735727311Writing region close event to WAL at 1731735727311Running coprocessor post-close hooks at 1731735727314 (+3 ms)Closed at 1731735727314 2024-11-16T05:42:07,315 WARN [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: ebfb6a6d32a314c2093cd1145e998024 to self. 2024-11-16T05:42:07,316 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,316 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=ebfb6a6d32a314c2093cd1145e998024, regionState=CLOSED 2024-11-16T05:42:07,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:42:07,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-16T05:42:07,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 in 159 msec 2024-11-16T05:42:07,320 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, REOPEN/MOVE; state=CLOSED, location=3456ee6a3164,35555,1731735575219; forceNewPlan=false, retain=true 2024-11-16T05:42:07,470 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=ebfb6a6d32a314c2093cd1145e998024, regionState=OPENING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, REOPEN/MOVE because future has completed 2024-11-16T05:42:07,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:42:07,626 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:07,626 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:42:07,627 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,627 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:42:07,627 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,627 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,628 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,628 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebfb6a6d32a314c2093cd1145e998024 columnFamilyName A 2024-11-16T05:42:07,630 DEBUG [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:07,630 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(327): Store=ebfb6a6d32a314c2093cd1145e998024/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:42:07,630 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,631 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebfb6a6d32a314c2093cd1145e998024 columnFamilyName B 2024-11-16T05:42:07,631 DEBUG [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:07,631 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(327): Store=ebfb6a6d32a314c2093cd1145e998024/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:42:07,631 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,632 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebfb6a6d32a314c2093cd1145e998024 columnFamilyName C 2024-11-16T05:42:07,632 DEBUG [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:07,632 INFO [StoreOpener-ebfb6a6d32a314c2093cd1145e998024-1 {}] regionserver.HStore(327): Store=ebfb6a6d32a314c2093cd1145e998024/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:42:07,632 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,633 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,634 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,634 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,634 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,635 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:42:07,636 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,636 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened ebfb6a6d32a314c2093cd1145e998024; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63650127, jitterRate=-0.05153919756412506}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:42:07,636 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,637 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for ebfb6a6d32a314c2093cd1145e998024: Running coprocessor pre-open hook at 1731735727627Writing region info on filesystem at 1731735727627Initializing all the Stores at 1731735727627Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735727627Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735727628 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731735727628Cleaning up temporary data from old regions at 1731735727634 (+6 ms)Running coprocessor post-open hooks at 1731735727636 (+2 ms)Region opened successfully at 1731735727637 (+1 ms) 2024-11-16T05:42:07,638 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., pid=155, masterSystemTime=1731735727624 2024-11-16T05:42:07,639 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:07,639 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:07,640 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=ebfb6a6d32a314c2093cd1145e998024, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:42:07,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-16T05:42:07,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 in 169 msec 2024-11-16T05:42:07,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-16T05:42:07,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, REOPEN/MOVE in 487 msec 2024-11-16T05:42:07,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-16T05:42:07,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-16T05:42:07,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-16T05:42:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-16T05:42:07,648 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:07,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:07,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:07,650 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cc2f848, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:07,651 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:07,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@560eb478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:07,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:07,653 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d91a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:07,654 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@473be292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:07,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:07,655 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3568b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:07,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c58116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:07,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:07,658 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f47a837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:07,659 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:07,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6070ba63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:07,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:07,661 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c220e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:07,662 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40609b58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:07,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:07,664 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3684062a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:07,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce5575a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:07,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:07,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f98ce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:07,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e16a906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:07,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:07,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9e9259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:07,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:07,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:07,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339a66ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:07,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:07,674 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6db6ec90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:07,675 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675dd9a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,39409,-1 for getting cluster id 2024-11-16T05:42:07,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:42:07,677 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c99181d9-1a27-4d42-b12d-1ee1970e5198' 2024-11-16T05:42:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:42:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c99181d9-1a27-4d42-b12d-1ee1970e5198" 2024-11-16T05:42:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121a546b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,39409,-1] 2024-11-16T05:42:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:42:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:07,679 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:42:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f8aeb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:42:07,681 DEBUG [Thread-2418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:07,681 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:07,681 DEBUG [Thread-2426 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:07,681 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:07,681 DEBUG [Thread-2422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:07,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:07,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:07,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:07,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:07,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:42:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:42:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:42:07,685 DEBUG [Thread-2424 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:42:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:42:07,686 DEBUG [Thread-2435 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:07,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:42:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:07,687 DEBUG [Thread-2433 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-16T05:42:07,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:07,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T05:42:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:42:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:42:07,690 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:42:07,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:07,692 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:42:07,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:42:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:42:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:07,695 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:42:07,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T05:42:07,697 DEBUG [Thread-2437 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:07,697 DEBUG [Thread-2431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:42:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,35555,1731735575219, seqNum=-1] 2024-11-16T05:42:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:42:07,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:07,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:42:07,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:42:07,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5] 2024-11-16T05:42:07,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735787731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:07,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40626 deadline: 1731735787733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40624 deadline: 1731735787734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:07,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735787736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735787736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:07,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a65584b00fce49eeb6ad9ec63f29c97c_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735727694/Put/seqid=0 2024-11-16T05:42:07,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742369_1545 (size=12154) 2024-11-16T05:42:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T05:42:07,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:07,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:07,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:07,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:07,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:07,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735787848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40626 deadline: 1731735787849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40624 deadline: 1731735787849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:07,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735787849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735787849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:07,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T05:42:08,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735788058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40626 deadline: 1731735788058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40624 deadline: 1731735788058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735788058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735788059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,155 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:08,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a65584b00fce49eeb6ad9ec63f29c97c_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a65584b00fce49eeb6ad9ec63f29c97c_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:08,158 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/972c62cf5b854c6f9fece53eaef1e529, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:08,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/972c62cf5b854c6f9fece53eaef1e529 is 175, key is test_row_0/A:col10/1731735727694/Put/seqid=0 2024-11-16T05:42:08,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742370_1546 (size=30955) 2024-11-16T05:42:08,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:08,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:08,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T05:42:08,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735788368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40626 deadline: 1731735788368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40624 deadline: 1731735788368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735788368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735788368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:08,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:08,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,562 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/972c62cf5b854c6f9fece53eaef1e529 2024-11-16T05:42:08,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/14301cc129d8463ca4962ccfb4862d3b is 50, key is test_row_0/B:col10/1731735727694/Put/seqid=0 2024-11-16T05:42:08,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742371_1547 (size=12001) 2024-11-16T05:42:08,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:08,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:08,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:08,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:08,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T05:42:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735788878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40626 deadline: 1731735788878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40624 deadline: 1731735788878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735788878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735788879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:08,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:08,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:08,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:08,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:08,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/14301cc129d8463ca4962ccfb4862d3b 2024-11-16T05:42:09,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/faeb604d617944f9be9acb77245e077d is 50, key is test_row_0/C:col10/1731735727694/Put/seqid=0 2024-11-16T05:42:09,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742372_1548 (size=12001) 2024-11-16T05:42:09,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:09,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:09,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:09,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:09,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:09,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:09,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:09,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:09,291 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:42:09,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:09,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:09,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:09,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:09,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:09,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:09,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/faeb604d617944f9be9acb77245e077d 2024-11-16T05:42:09,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/972c62cf5b854c6f9fece53eaef1e529 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/972c62cf5b854c6f9fece53eaef1e529 2024-11-16T05:42:09,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/972c62cf5b854c6f9fece53eaef1e529, entries=150, sequenceid=18, filesize=30.2 K 2024-11-16T05:42:09,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/14301cc129d8463ca4962ccfb4862d3b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/14301cc129d8463ca4962ccfb4862d3b 2024-11-16T05:42:09,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/14301cc129d8463ca4962ccfb4862d3b, entries=150, sequenceid=18, filesize=11.7 K 2024-11-16T05:42:09,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/faeb604d617944f9be9acb77245e077d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/faeb604d617944f9be9acb77245e077d 2024-11-16T05:42:09,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/faeb604d617944f9be9acb77245e077d, entries=150, sequenceid=18, filesize=11.7 K 2024-11-16T05:42:09,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for ebfb6a6d32a314c2093cd1145e998024 in 1729ms, sequenceid=18, compaction requested=false 2024-11-16T05:42:09,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:09,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T05:42:09,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:09,526 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T05:42:09,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116067dfb0de82845c483c1434e4390f2d9_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735727733/Put/seqid=0 2024-11-16T05:42:09,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742373_1549 (size=12154) 2024-11-16T05:42:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:09,550 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116067dfb0de82845c483c1434e4390f2d9_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116067dfb0de82845c483c1434e4390f2d9_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/b99b23db4fc9444e9fe13e2912852271, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/b99b23db4fc9444e9fe13e2912852271 is 175, key is test_row_0/A:col10/1731735727733/Put/seqid=0 2024-11-16T05:42:09,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742374_1550 (size=30955) 2024-11-16T05:42:09,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T05:42:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:09,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40624 deadline: 1731735789898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40626 deadline: 1731735789898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:09,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735789899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:09,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735789899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:09,902 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_2' in ebfb6a6d32a314c2093cd1145e998024 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:42:09,902 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in ebfb6a6d32a314c2093cd1145e998024 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:09,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735789902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:09,954 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/b99b23db4fc9444e9fe13e2912852271 2024-11-16T05:42:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/dcfe3666547244ad821a3dd7a8bdcf38 is 50, key is test_row_0/B:col10/1731735727733/Put/seqid=0 2024-11-16T05:42:09,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742375_1551 (size=12001) 2024-11-16T05:42:09,970 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/dcfe3666547244ad821a3dd7a8bdcf38 2024-11-16T05:42:09,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/36dddfb4443d4c51a8bad92c90452fc8 is 50, key is test_row_0/C:col10/1731735727733/Put/seqid=0 2024-11-16T05:42:09,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742376_1552 (size=12001) 2024-11-16T05:42:10,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735790008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735790008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735790008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735790218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735790218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735790218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,399 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/36dddfb4443d4c51a8bad92c90452fc8 2024-11-16T05:42:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/b99b23db4fc9444e9fe13e2912852271 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b99b23db4fc9444e9fe13e2912852271 2024-11-16T05:42:10,416 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b99b23db4fc9444e9fe13e2912852271, entries=150, sequenceid=41, filesize=30.2 K 2024-11-16T05:42:10,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/dcfe3666547244ad821a3dd7a8bdcf38 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dcfe3666547244ad821a3dd7a8bdcf38 2024-11-16T05:42:10,420 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dcfe3666547244ad821a3dd7a8bdcf38, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T05:42:10,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/36dddfb4443d4c51a8bad92c90452fc8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/36dddfb4443d4c51a8bad92c90452fc8 2024-11-16T05:42:10,424 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/36dddfb4443d4c51a8bad92c90452fc8, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T05:42:10,424 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ebfb6a6d32a314c2093cd1145e998024 in 898ms, sequenceid=41, compaction requested=false 2024-11-16T05:42:10,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:10,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:10,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-16T05:42:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-16T05:42:10,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-16T05:42:10,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7340 sec 2024-11-16T05:42:10,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.7420 sec 2024-11-16T05:42:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:10,530 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:42:10,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T05:42:10,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fd32beb2bf354e79b896a385e921f209_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735729901/Put/seqid=0 2024-11-16T05:42:10,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742377_1553 (size=12154) 2024-11-16T05:42:10,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735790558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735790558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735790559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735790668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735790668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735790668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735790878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735790878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735790878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:10,941 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:10,943 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fd32beb2bf354e79b896a385e921f209_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fd32beb2bf354e79b896a385e921f209_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:10,944 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/99eb4514e0d8469ab10e879cd11242cb, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:10,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/99eb4514e0d8469ab10e879cd11242cb is 175, key is test_row_0/A:col10/1731735729901/Put/seqid=0 2024-11-16T05:42:10,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742378_1554 (size=30955) 2024-11-16T05:42:11,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735791188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:11,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735791188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735791188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:11,349 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/99eb4514e0d8469ab10e879cd11242cb 2024-11-16T05:42:11,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/c0a5a80bb93045b989c0e240eec3ac08 is 50, key is test_row_0/B:col10/1731735729901/Put/seqid=0 2024-11-16T05:42:11,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742379_1555 (size=12001) 2024-11-16T05:42:11,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735791698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:11,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735791698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735791698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:11,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/c0a5a80bb93045b989c0e240eec3ac08 2024-11-16T05:42:11,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/83ee8f6005fa403c8d59efe975f504c6 is 50, key is test_row_0/C:col10/1731735729901/Put/seqid=0 2024-11-16T05:42:11,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742380_1556 (size=12001) 2024-11-16T05:42:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T05:42:11,858 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:11,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:42:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-16T05:42:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T05:42:11,860 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:42:11,860 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:42:11,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:42:11,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40624 deadline: 1731735791908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:11,911 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_2' in ebfb6a6d32a314c2093cd1145e998024 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:11,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40626 deadline: 1731735791931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:11,933 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in ebfb6a6d32a314c2093cd1145e998024 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:42:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T05:42:12,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T05:42:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:12,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:12,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/83ee8f6005fa403c8d59efe975f504c6 2024-11-16T05:42:12,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T05:42:12,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:12,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:12,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/99eb4514e0d8469ab10e879cd11242cb as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/99eb4514e0d8469ab10e879cd11242cb 2024-11-16T05:42:12,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/99eb4514e0d8469ab10e879cd11242cb, entries=150, sequenceid=56, filesize=30.2 K 2024-11-16T05:42:12,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/c0a5a80bb93045b989c0e240eec3ac08 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c0a5a80bb93045b989c0e240eec3ac08 2024-11-16T05:42:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T05:42:12,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c0a5a80bb93045b989c0e240eec3ac08, entries=150, sequenceid=56, filesize=11.7 K 2024-11-16T05:42:12,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/83ee8f6005fa403c8d59efe975f504c6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/83ee8f6005fa403c8d59efe975f504c6 2024-11-16T05:42:12,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/83ee8f6005fa403c8d59efe975f504c6, entries=150, sequenceid=56, filesize=11.7 K 2024-11-16T05:42:12,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ebfb6a6d32a314c2093cd1145e998024 in 1656ms, sequenceid=56, compaction requested=true 2024-11-16T05:42:12,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:42:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:42:12,187 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:12,187 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:42:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:12,188 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:12,188 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:12,188 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/A is initiating minor compaction (all files) 2024-11-16T05:42:12,188 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/B is initiating minor compaction (all files) 2024-11-16T05:42:12,188 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/B in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:12,188 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/A in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:12,188 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/14301cc129d8463ca4962ccfb4862d3b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dcfe3666547244ad821a3dd7a8bdcf38, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c0a5a80bb93045b989c0e240eec3ac08] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=35.2 K 2024-11-16T05:42:12,188 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/972c62cf5b854c6f9fece53eaef1e529, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b99b23db4fc9444e9fe13e2912852271, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/99eb4514e0d8469ab10e879cd11242cb] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=90.7 K 2024-11-16T05:42:12,188 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:12,188 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/972c62cf5b854c6f9fece53eaef1e529, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b99b23db4fc9444e9fe13e2912852271, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/99eb4514e0d8469ab10e879cd11242cb] 2024-11-16T05:42:12,188 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 14301cc129d8463ca4962ccfb4862d3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731735727693 2024-11-16T05:42:12,188 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 972c62cf5b854c6f9fece53eaef1e529, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731735727693 2024-11-16T05:42:12,188 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting dcfe3666547244ad821a3dd7a8bdcf38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731735727731 2024-11-16T05:42:12,188 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b99b23db4fc9444e9fe13e2912852271, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731735727731 2024-11-16T05:42:12,189 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting c0a5a80bb93045b989c0e240eec3ac08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731735729901 2024-11-16T05:42:12,189 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99eb4514e0d8469ab10e879cd11242cb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731735729901 2024-11-16T05:42:12,194 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:12,195 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:12,196 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111626d1c1f59f6e44e8be632c90ac4d884a_ebfb6a6d32a314c2093cd1145e998024 store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:12,196 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/e882c9ab684a4c86bb2d2134124c8d03 is 50, key is test_row_0/B:col10/1731735729901/Put/seqid=0 2024-11-16T05:42:12,197 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111626d1c1f59f6e44e8be632c90ac4d884a_ebfb6a6d32a314c2093cd1145e998024, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:12,198 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111626d1c1f59f6e44e8be632c90ac4d884a_ebfb6a6d32a314c2093cd1145e998024 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:12,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742381_1557 (size=12104) 2024-11-16T05:42:12,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742382_1558 (size=4469) 2024-11-16T05:42:12,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T05:42:12,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:12,321 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T05:42:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d4dd71a43d4f42d1b496ca7fd46e67d9_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735730558/Put/seqid=0 2024-11-16T05:42:12,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742383_1559 (size=12154) 2024-11-16T05:42:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T05:42:12,602 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:12,602 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/e4a08cfa56a24b11b55a26f7dda6472e is 175, key is test_row_0/A:col10/1731735729901/Put/seqid=0 2024-11-16T05:42:12,603 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/e882c9ab684a4c86bb2d2134124c8d03 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e882c9ab684a4c86bb2d2134124c8d03 2024-11-16T05:42:12,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742384_1560 (size=31058) 2024-11-16T05:42:12,607 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/B of ebfb6a6d32a314c2093cd1145e998024 into e882c9ab684a4c86bb2d2134124c8d03(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:12,607 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:12,607 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/B, priority=13, startTime=1731735732187; duration=0sec 2024-11-16T05:42:12,607 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:12,607 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:B 2024-11-16T05:42:12,607 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:12,608 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:12,608 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/C is initiating minor compaction (all files) 2024-11-16T05:42:12,608 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/C in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:12,608 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/faeb604d617944f9be9acb77245e077d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/36dddfb4443d4c51a8bad92c90452fc8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/83ee8f6005fa403c8d59efe975f504c6] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=35.2 K 2024-11-16T05:42:12,609 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting faeb604d617944f9be9acb77245e077d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731735727693 2024-11-16T05:42:12,609 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 36dddfb4443d4c51a8bad92c90452fc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731735727731 2024-11-16T05:42:12,609 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 83ee8f6005fa403c8d59efe975f504c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731735729901 2024-11-16T05:42:12,614 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:12,614 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/de9f5b6234b541ecb0aff73e8bd70c5f is 50, key is test_row_0/C:col10/1731735729901/Put/seqid=0 2024-11-16T05:42:12,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742385_1561 (size=12104) 2024-11-16T05:42:12,620 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/de9f5b6234b541ecb0aff73e8bd70c5f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/de9f5b6234b541ecb0aff73e8bd70c5f 2024-11-16T05:42:12,623 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/C of ebfb6a6d32a314c2093cd1145e998024 into de9f5b6234b541ecb0aff73e8bd70c5f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:12,623 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:12,623 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/C, priority=13, startTime=1731735732187; duration=0sec 2024-11-16T05:42:12,623 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:12,623 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:C 2024-11-16T05:42:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:12,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:12,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735792723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:12,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735792725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735792726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:12,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:12,735 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d4dd71a43d4f42d1b496ca7fd46e67d9_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d4dd71a43d4f42d1b496ca7fd46e67d9_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/55add32ccc1f4210bca7937ac9cef55a, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/55add32ccc1f4210bca7937ac9cef55a is 175, key is test_row_0/A:col10/1731735730558/Put/seqid=0 2024-11-16T05:42:12,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742386_1562 (size=30955) 2024-11-16T05:42:12,740 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/55add32ccc1f4210bca7937ac9cef55a 2024-11-16T05:42:12,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/d7de74cf05004debb27ad7c1e903510b is 50, key is test_row_0/B:col10/1731735730558/Put/seqid=0 2024-11-16T05:42:12,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742387_1563 (size=12001) 2024-11-16T05:42:12,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735792838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:12,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735792838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735792838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T05:42:13,010 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/e4a08cfa56a24b11b55a26f7dda6472e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/e4a08cfa56a24b11b55a26f7dda6472e 2024-11-16T05:42:13,015 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/A of ebfb6a6d32a314c2093cd1145e998024 into e4a08cfa56a24b11b55a26f7dda6472e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:13,015 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:13,015 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/A, priority=13, startTime=1731735732186; duration=0sec 2024-11-16T05:42:13,015 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:13,015 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:A 2024-11-16T05:42:13,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735793048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:13,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735793048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:13,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735793048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:13,148 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/d7de74cf05004debb27ad7c1e903510b 2024-11-16T05:42:13,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/003d0af8163f4f4d8c4b208c7b785751 is 50, key is test_row_0/C:col10/1731735730558/Put/seqid=0 2024-11-16T05:42:13,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742388_1564 (size=12001) 2024-11-16T05:42:13,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735793358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:13,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735793358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:13,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:13,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735793358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:13,558 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/003d0af8163f4f4d8c4b208c7b785751 2024-11-16T05:42:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/55add32ccc1f4210bca7937ac9cef55a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/55add32ccc1f4210bca7937ac9cef55a 2024-11-16T05:42:13,565 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/55add32ccc1f4210bca7937ac9cef55a, entries=150, sequenceid=77, filesize=30.2 K 2024-11-16T05:42:13,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/d7de74cf05004debb27ad7c1e903510b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/d7de74cf05004debb27ad7c1e903510b 2024-11-16T05:42:13,568 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/d7de74cf05004debb27ad7c1e903510b, entries=150, sequenceid=77, filesize=11.7 K 2024-11-16T05:42:13,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/003d0af8163f4f4d8c4b208c7b785751 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/003d0af8163f4f4d8c4b208c7b785751 2024-11-16T05:42:13,572 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/003d0af8163f4f4d8c4b208c7b785751, entries=150, sequenceid=77, filesize=11.7 K 2024-11-16T05:42:13,572 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for ebfb6a6d32a314c2093cd1145e998024 in 1251ms, sequenceid=77, compaction requested=false 2024-11-16T05:42:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:13,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:13,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-16T05:42:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-16T05:42:13,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-16T05:42:13,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7140 sec 2024-11-16T05:42:13,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.7180 sec 2024-11-16T05:42:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:13,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:42:13,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T05:42:13,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116658e82881814499799114d81ecd357ff_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735733871/Put/seqid=0 2024-11-16T05:42:13,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742389_1565 (size=17034) 2024-11-16T05:42:13,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:13,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116658e82881814499799114d81ecd357ff_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116658e82881814499799114d81ecd357ff_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735793898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:13,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/54b8cd6b81b246cda5cab4bb2a7ac304, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735793899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/54b8cd6b81b246cda5cab4bb2a7ac304 is 175, key is test_row_0/A:col10/1731735733871/Put/seqid=0 2024-11-16T05:42:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:13,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735793899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:13,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742390_1566 (size=48139) 2024-11-16T05:42:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T05:42:14,008 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:14,009 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:42:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-16T05:42:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T05:42:14,010 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:42:14,011 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:42:14,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:42:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735794008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735794008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735794008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T05:42:14,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T05:42:14,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:14,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735794218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:14,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735794218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:14,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735794218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:14,316 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/54b8cd6b81b246cda5cab4bb2a7ac304 2024-11-16T05:42:14,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T05:42:14,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:14,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/20b603f699d14d8f941ae933452d276b is 50, key is test_row_0/B:col10/1731735733871/Put/seqid=0 2024-11-16T05:42:14,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742391_1567 (size=12001) 2024-11-16T05:42:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T05:42:14,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T05:42:14,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735794528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:14,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:14,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735794528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735794528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:14,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T05:42:14,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:14,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T05:42:14,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/20b603f699d14d8f941ae933452d276b 2024-11-16T05:42:14,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/b054c854af5e40fd894d2d305f51b4c7 is 50, key is test_row_0/C:col10/1731735733871/Put/seqid=0 2024-11-16T05:42:14,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742392_1568 (size=12001) 2024-11-16T05:42:14,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/b054c854af5e40fd894d2d305f51b4c7 2024-11-16T05:42:14,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/54b8cd6b81b246cda5cab4bb2a7ac304 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/54b8cd6b81b246cda5cab4bb2a7ac304 2024-11-16T05:42:14,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/54b8cd6b81b246cda5cab4bb2a7ac304, entries=250, sequenceid=98, filesize=47.0 K 2024-11-16T05:42:14,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/20b603f699d14d8f941ae933452d276b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/20b603f699d14d8f941ae933452d276b 2024-11-16T05:42:14,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/20b603f699d14d8f941ae933452d276b, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T05:42:14,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/b054c854af5e40fd894d2d305f51b4c7 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/b054c854af5e40fd894d2d305f51b4c7 2024-11-16T05:42:14,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/b054c854af5e40fd894d2d305f51b4c7, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T05:42:14,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for ebfb6a6d32a314c2093cd1145e998024 in 878ms, sequenceid=98, compaction requested=true 2024-11-16T05:42:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:42:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:14,751 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:42:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:14,751 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:42:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:14,751 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:14,752 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/A is initiating minor compaction (all files) 2024-11-16T05:42:14,752 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:14,752 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/B is initiating minor compaction (all files) 2024-11-16T05:42:14,752 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/A in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,752 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/B in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,752 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e882c9ab684a4c86bb2d2134124c8d03, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/d7de74cf05004debb27ad7c1e903510b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/20b603f699d14d8f941ae933452d276b] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=35.3 K 2024-11-16T05:42:14,752 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/e4a08cfa56a24b11b55a26f7dda6472e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/55add32ccc1f4210bca7937ac9cef55a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/54b8cd6b81b246cda5cab4bb2a7ac304] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=107.6 K 2024-11-16T05:42:14,752 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,752 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/e4a08cfa56a24b11b55a26f7dda6472e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/55add32ccc1f4210bca7937ac9cef55a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/54b8cd6b81b246cda5cab4bb2a7ac304] 2024-11-16T05:42:14,752 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e882c9ab684a4c86bb2d2134124c8d03, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731735729901 2024-11-16T05:42:14,752 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4a08cfa56a24b11b55a26f7dda6472e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731735729901 2024-11-16T05:42:14,752 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting d7de74cf05004debb27ad7c1e903510b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731735730545 2024-11-16T05:42:14,752 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55add32ccc1f4210bca7937ac9cef55a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731735730545 2024-11-16T05:42:14,753 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b603f699d14d8f941ae933452d276b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731735732725 2024-11-16T05:42:14,753 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54b8cd6b81b246cda5cab4bb2a7ac304, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731735732722 2024-11-16T05:42:14,759 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#B#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:14,759 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/f94ee81ef7b9438aa5e1e355f7964cc8 is 50, key is test_row_0/B:col10/1731735733871/Put/seqid=0 2024-11-16T05:42:14,760 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:14,762 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411166fba8b77a4db41b991a7075dff721e18_ebfb6a6d32a314c2093cd1145e998024 store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:14,764 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411166fba8b77a4db41b991a7075dff721e18_ebfb6a6d32a314c2093cd1145e998024, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:14,764 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166fba8b77a4db41b991a7075dff721e18_ebfb6a6d32a314c2093cd1145e998024 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:14,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742393_1569 (size=12207) 2024-11-16T05:42:14,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742394_1570 (size=4469) 2024-11-16T05:42:14,770 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/f94ee81ef7b9438aa5e1e355f7964cc8 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f94ee81ef7b9438aa5e1e355f7964cc8 2024-11-16T05:42:14,773 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/B of ebfb6a6d32a314c2093cd1145e998024 into f94ee81ef7b9438aa5e1e355f7964cc8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:14,773 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:14,774 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/B, priority=13, startTime=1731735734751; duration=0sec 2024-11-16T05:42:14,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:14,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:B 2024-11-16T05:42:14,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:14,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:14,774 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/C is initiating minor compaction (all files) 2024-11-16T05:42:14,774 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/C in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,775 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/de9f5b6234b541ecb0aff73e8bd70c5f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/003d0af8163f4f4d8c4b208c7b785751, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/b054c854af5e40fd894d2d305f51b4c7] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=35.3 K 2024-11-16T05:42:14,775 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting de9f5b6234b541ecb0aff73e8bd70c5f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731735729901 2024-11-16T05:42:14,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T05:42:14,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:14,775 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T05:42:14,776 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 003d0af8163f4f4d8c4b208c7b785751, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731735730545 2024-11-16T05:42:14,776 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting b054c854af5e40fd894d2d305f51b4c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731735732725 2024-11-16T05:42:14,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165c1b8d491e4742efad7cf258760e5fec_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735733891/Put/seqid=0 2024-11-16T05:42:14,781 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#C#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:14,781 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/e5016c0229944aedbd23081c3efff437 is 50, key is test_row_0/C:col10/1731735733871/Put/seqid=0 2024-11-16T05:42:14,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742395_1571 (size=12154) 2024-11-16T05:42:14,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:14,788 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165c1b8d491e4742efad7cf258760e5fec_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c1b8d491e4742efad7cf258760e5fec_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:14,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/3f90395abb4146529a2703ad13ccb6c6, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:14,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/3f90395abb4146529a2703ad13ccb6c6 is 175, key is test_row_0/A:col10/1731735733891/Put/seqid=0 2024-11-16T05:42:14,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742396_1572 (size=12207) 2024-11-16T05:42:14,802 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/e5016c0229944aedbd23081c3efff437 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/e5016c0229944aedbd23081c3efff437 2024-11-16T05:42:14,806 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/C of ebfb6a6d32a314c2093cd1145e998024 into e5016c0229944aedbd23081c3efff437(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:14,806 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:14,806 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/C, priority=13, startTime=1731735734751; duration=0sec 2024-11-16T05:42:14,806 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:14,806 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:C 2024-11-16T05:42:14,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742397_1573 (size=30955) 2024-11-16T05:42:14,949 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:42:14,949 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T05:42:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:15,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735795063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735795066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735795067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T05:42:15,168 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:15,168 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/01082936c37a406bb5a9216d5a1c6066 is 175, key is test_row_0/A:col10/1731735733871/Put/seqid=0 2024-11-16T05:42:15,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742398_1574 (size=31161) 2024-11-16T05:42:15,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735795178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735795178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735795178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,211 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/3f90395abb4146529a2703ad13ccb6c6 2024-11-16T05:42:15,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/4819f5b610ae4c7f9213d353a3dd573a is 50, key is test_row_0/B:col10/1731735733891/Put/seqid=0 2024-11-16T05:42:15,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742399_1575 (size=12001) 2024-11-16T05:42:15,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735795388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735795388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735795388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,575 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/01082936c37a406bb5a9216d5a1c6066 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/01082936c37a406bb5a9216d5a1c6066 2024-11-16T05:42:15,578 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/A of ebfb6a6d32a314c2093cd1145e998024 into 01082936c37a406bb5a9216d5a1c6066(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:15,578 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:15,578 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/A, priority=13, startTime=1731735734751; duration=0sec 2024-11-16T05:42:15,578 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:15,578 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:A 2024-11-16T05:42:15,620 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/4819f5b610ae4c7f9213d353a3dd573a 2024-11-16T05:42:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/9b2c1836443340da9584203a1db1796c is 50, key is test_row_0/C:col10/1731735733891/Put/seqid=0 2024-11-16T05:42:15,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742400_1576 (size=12001) 2024-11-16T05:42:15,628 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/9b2c1836443340da9584203a1db1796c 2024-11-16T05:42:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/3f90395abb4146529a2703ad13ccb6c6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/3f90395abb4146529a2703ad13ccb6c6 2024-11-16T05:42:15,634 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/3f90395abb4146529a2703ad13ccb6c6, entries=150, sequenceid=117, filesize=30.2 K 2024-11-16T05:42:15,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/4819f5b610ae4c7f9213d353a3dd573a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4819f5b610ae4c7f9213d353a3dd573a 2024-11-16T05:42:15,640 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4819f5b610ae4c7f9213d353a3dd573a, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T05:42:15,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/9b2c1836443340da9584203a1db1796c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9b2c1836443340da9584203a1db1796c 2024-11-16T05:42:15,644 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9b2c1836443340da9584203a1db1796c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T05:42:15,644 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for ebfb6a6d32a314c2093cd1145e998024 in 869ms, sequenceid=117, compaction requested=false 2024-11-16T05:42:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-16T05:42:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-16T05:42:15,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-16T05:42:15,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6350 sec 2024-11-16T05:42:15,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.6380 sec 2024-11-16T05:42:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:15,700 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:42:15,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T05:42:15,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116da2afdec4fb2438b83e81352ff4b24ff_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735735701/Put/seqid=0 2024-11-16T05:42:15,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742401_1577 (size=14794) 2024-11-16T05:42:15,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735795720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735795721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735795722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735795828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735795838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735795838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40624 deadline: 1731735795928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,930 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_2' in ebfb6a6d32a314c2093cd1145e998024 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:42:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40626 deadline: 1731735795958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:15,960 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in ebfb6a6d32a314c2093cd1145e998024 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:42:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:16,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735796038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735796048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735796048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:16,119 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:16,121 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116da2afdec4fb2438b83e81352ff4b24ff_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da2afdec4fb2438b83e81352ff4b24ff_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:16,129 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/c1b60b6aa8654442ac260c0769994121, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:16,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/c1b60b6aa8654442ac260c0769994121 is 175, key is test_row_0/A:col10/1731735735701/Put/seqid=0 2024-11-16T05:42:16,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742402_1578 (size=39749) 2024-11-16T05:42:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T05:42:16,158 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:16,158 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:42:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-16T05:42:16,160 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:42:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T05:42:16,160 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:42:16,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:42:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T05:42:16,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T05:42:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:16,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735796348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:16,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735796358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:16,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735796358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:16,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T05:42:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T05:42:16,533 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/c1b60b6aa8654442ac260c0769994121 2024-11-16T05:42:16,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/b733099e52994abc8b4b7364e49af095 is 50, key is test_row_0/B:col10/1731735735701/Put/seqid=0 2024-11-16T05:42:16,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742403_1579 (size=12151) 2024-11-16T05:42:16,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/b733099e52994abc8b4b7364e49af095 2024-11-16T05:42:16,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/6e819b3883a44109bb3a88e6503d0599 is 50, key is test_row_0/C:col10/1731735735701/Put/seqid=0 2024-11-16T05:42:16,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742404_1580 (size=12151) 2024-11-16T05:42:16,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T05:42:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T05:42:16,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:16,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T05:42:16,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735796858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:16,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735796868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:16,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735796868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:16,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T05:42:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:16,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/6e819b3883a44109bb3a88e6503d0599 2024-11-16T05:42:16,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/c1b60b6aa8654442ac260c0769994121 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/c1b60b6aa8654442ac260c0769994121 2024-11-16T05:42:16,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/c1b60b6aa8654442ac260c0769994121, entries=200, sequenceid=140, filesize=38.8 K 2024-11-16T05:42:16,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/b733099e52994abc8b4b7364e49af095 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/b733099e52994abc8b4b7364e49af095 2024-11-16T05:42:16,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/b733099e52994abc8b4b7364e49af095, entries=150, sequenceid=140, filesize=11.9 K 2024-11-16T05:42:16,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/6e819b3883a44109bb3a88e6503d0599 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/6e819b3883a44109bb3a88e6503d0599 2024-11-16T05:42:16,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/6e819b3883a44109bb3a88e6503d0599, entries=150, sequenceid=140, filesize=11.9 K 2024-11-16T05:42:16,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for ebfb6a6d32a314c2093cd1145e998024 in 1263ms, sequenceid=140, compaction requested=true 2024-11-16T05:42:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:42:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:16,964 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:42:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:16,964 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:42:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:16,964 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:16,964 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:16,964 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/A is initiating minor compaction (all files) 2024-11-16T05:42:16,964 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/B is initiating minor compaction (all files) 2024-11-16T05:42:16,964 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/A in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,965 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/B in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,965 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/01082936c37a406bb5a9216d5a1c6066, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/3f90395abb4146529a2703ad13ccb6c6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/c1b60b6aa8654442ac260c0769994121] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=99.5 K 2024-11-16T05:42:16,965 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f94ee81ef7b9438aa5e1e355f7964cc8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4819f5b610ae4c7f9213d353a3dd573a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/b733099e52994abc8b4b7364e49af095] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=35.5 K 2024-11-16T05:42:16,965 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,965 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/01082936c37a406bb5a9216d5a1c6066, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/3f90395abb4146529a2703ad13ccb6c6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/c1b60b6aa8654442ac260c0769994121] 2024-11-16T05:42:16,965 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f94ee81ef7b9438aa5e1e355f7964cc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731735732725 2024-11-16T05:42:16,965 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01082936c37a406bb5a9216d5a1c6066, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731735732725 2024-11-16T05:42:16,965 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4819f5b610ae4c7f9213d353a3dd573a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731735733891 2024-11-16T05:42:16,965 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f90395abb4146529a2703ad13ccb6c6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731735733891 2024-11-16T05:42:16,966 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting b733099e52994abc8b4b7364e49af095, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731735735058 2024-11-16T05:42:16,966 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1b60b6aa8654442ac260c0769994121, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731735735058 2024-11-16T05:42:16,972 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:16,973 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#B#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:16,973 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/e42eabbf4cb14c72902cf4b7f79b91d4 is 50, key is test_row_0/B:col10/1731735735701/Put/seqid=0 2024-11-16T05:42:16,975 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111646dff43685a0415ebf555b2becc11a0a_ebfb6a6d32a314c2093cd1145e998024 store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:16,977 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111646dff43685a0415ebf555b2becc11a0a_ebfb6a6d32a314c2093cd1145e998024, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:16,978 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111646dff43685a0415ebf555b2becc11a0a_ebfb6a6d32a314c2093cd1145e998024 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:16,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742405_1581 (size=12459) 2024-11-16T05:42:16,988 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/e42eabbf4cb14c72902cf4b7f79b91d4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e42eabbf4cb14c72902cf4b7f79b91d4 2024-11-16T05:42:16,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742406_1582 (size=4469) 2024-11-16T05:42:16,996 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#A#compaction#492 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:16,996 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/1bf831f450d94a5083971c176208f16c is 175, key is test_row_0/A:col10/1731735735701/Put/seqid=0 2024-11-16T05:42:16,998 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/B of ebfb6a6d32a314c2093cd1145e998024 into e42eabbf4cb14c72902cf4b7f79b91d4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:16,998 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:16,998 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/B, priority=13, startTime=1731735736964; duration=0sec 2024-11-16T05:42:16,998 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:16,998 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:B 2024-11-16T05:42:16,998 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:16,999 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:16,999 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/C is initiating minor compaction (all files) 2024-11-16T05:42:16,999 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/C in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:16,999 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/e5016c0229944aedbd23081c3efff437, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9b2c1836443340da9584203a1db1796c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/6e819b3883a44109bb3a88e6503d0599] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=35.5 K 2024-11-16T05:42:16,999 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e5016c0229944aedbd23081c3efff437, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731735732725 2024-11-16T05:42:16,999 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b2c1836443340da9584203a1db1796c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731735733891 2024-11-16T05:42:16,999 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e819b3883a44109bb3a88e6503d0599, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731735735058 2024-11-16T05:42:17,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742407_1583 (size=31413) 2024-11-16T05:42:17,007 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#C#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:17,008 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/12fff360a8924aca83c11ce66b3825a4 is 50, key is test_row_0/C:col10/1731735735701/Put/seqid=0 2024-11-16T05:42:17,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742408_1584 (size=12459) 2024-11-16T05:42:17,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T05:42:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:17,075 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T05:42:17,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164c734373f8d749099b5e6649f47150a2_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735735720/Put/seqid=0 2024-11-16T05:42:17,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742409_1585 (size=12304) 2024-11-16T05:42:17,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:17,090 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164c734373f8d749099b5e6649f47150a2_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164c734373f8d749099b5e6649f47150a2_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:17,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/2f64d5cb76064ecbb9a830483b577c37, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:17,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/2f64d5cb76064ecbb9a830483b577c37 is 175, key is test_row_0/A:col10/1731735735720/Put/seqid=0 2024-11-16T05:42:17,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742410_1586 (size=31105) 2024-11-16T05:42:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T05:42:17,420 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/1bf831f450d94a5083971c176208f16c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/1bf831f450d94a5083971c176208f16c 2024-11-16T05:42:17,421 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/12fff360a8924aca83c11ce66b3825a4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/12fff360a8924aca83c11ce66b3825a4 2024-11-16T05:42:17,424 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/A of ebfb6a6d32a314c2093cd1145e998024 into 1bf831f450d94a5083971c176208f16c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:17,424 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:17,424 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/A, priority=13, startTime=1731735736964; duration=0sec 2024-11-16T05:42:17,424 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:17,424 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:A 2024-11-16T05:42:17,431 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/C of ebfb6a6d32a314c2093cd1145e998024 into 12fff360a8924aca83c11ce66b3825a4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:17,432 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:17,432 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/C, priority=13, startTime=1731735736964; duration=0sec 2024-11-16T05:42:17,432 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:17,432 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:C 2024-11-16T05:42:17,496 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/2f64d5cb76064ecbb9a830483b577c37 2024-11-16T05:42:17,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/f7b60a1a16a948658b78f8adf8528603 is 50, key is test_row_0/B:col10/1731735735720/Put/seqid=0 2024-11-16T05:42:17,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742411_1587 (size=12151) 2024-11-16T05:42:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:17,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:17,906 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/f7b60a1a16a948658b78f8adf8528603 2024-11-16T05:42:17,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735797906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:17,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/208e74ac408d4bdb9f0b2464dd6364ef is 50, key is test_row_0/C:col10/1731735735720/Put/seqid=0 2024-11-16T05:42:17,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735797908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:17,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735797909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:17,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742412_1588 (size=12151) 2024-11-16T05:42:18,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735798018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735798019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735798019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735798228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735798228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735798228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T05:42:18,315 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/208e74ac408d4bdb9f0b2464dd6364ef 2024-11-16T05:42:18,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/2f64d5cb76064ecbb9a830483b577c37 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2f64d5cb76064ecbb9a830483b577c37 2024-11-16T05:42:18,321 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2f64d5cb76064ecbb9a830483b577c37, entries=150, sequenceid=156, filesize=30.4 K 2024-11-16T05:42:18,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/f7b60a1a16a948658b78f8adf8528603 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f7b60a1a16a948658b78f8adf8528603 2024-11-16T05:42:18,324 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f7b60a1a16a948658b78f8adf8528603, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T05:42:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/208e74ac408d4bdb9f0b2464dd6364ef as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/208e74ac408d4bdb9f0b2464dd6364ef 2024-11-16T05:42:18,327 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/208e74ac408d4bdb9f0b2464dd6364ef, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T05:42:18,328 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ebfb6a6d32a314c2093cd1145e998024 in 1253ms, sequenceid=156, compaction requested=false 2024-11-16T05:42:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-16T05:42:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-16T05:42:18,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-16T05:42:18,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1690 sec 2024-11-16T05:42:18,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.1730 sec 2024-11-16T05:42:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:18,541 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:42:18,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T05:42:18,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a43825c66c03455582286aeccf06020a_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735737908/Put/seqid=0 2024-11-16T05:42:18,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742413_1589 (size=17284) 2024-11-16T05:42:18,560 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:18,564 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a43825c66c03455582286aeccf06020a_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a43825c66c03455582286aeccf06020a_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:18,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735798561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,565 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/5a9797873f6e4133bad142d246ae6a22, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:18,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/5a9797873f6e4133bad142d246ae6a22 is 175, key is test_row_0/A:col10/1731735737908/Put/seqid=0 2024-11-16T05:42:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735798563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735798564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742414_1590 (size=48389) 2024-11-16T05:42:18,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735798668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735798678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735798678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735798878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735798888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735798888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:18,980 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/5a9797873f6e4133bad142d246ae6a22 2024-11-16T05:42:18,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/0a72f4bc7a1349d7b87ea06ecdaf8c6e is 50, key is test_row_0/B:col10/1731735737908/Put/seqid=0 2024-11-16T05:42:18,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742415_1591 (size=12151) 2024-11-16T05:42:19,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735799188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735799198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735799198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:19,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/0a72f4bc7a1349d7b87ea06ecdaf8c6e 2024-11-16T05:42:19,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/815c81b1055a43e6aeb1495ce75bf348 is 50, key is test_row_0/C:col10/1731735737908/Put/seqid=0 2024-11-16T05:42:19,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742416_1592 (size=12151) 2024-11-16T05:42:19,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735799698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:19,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735799707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:19,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735799708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:19,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/815c81b1055a43e6aeb1495ce75bf348 2024-11-16T05:42:19,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/5a9797873f6e4133bad142d246ae6a22 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/5a9797873f6e4133bad142d246ae6a22 2024-11-16T05:42:19,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/5a9797873f6e4133bad142d246ae6a22, entries=250, sequenceid=182, filesize=47.3 K 2024-11-16T05:42:19,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/0a72f4bc7a1349d7b87ea06ecdaf8c6e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/0a72f4bc7a1349d7b87ea06ecdaf8c6e 2024-11-16T05:42:19,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/0a72f4bc7a1349d7b87ea06ecdaf8c6e, entries=150, sequenceid=182, filesize=11.9 K 2024-11-16T05:42:19,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/815c81b1055a43e6aeb1495ce75bf348 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/815c81b1055a43e6aeb1495ce75bf348 2024-11-16T05:42:19,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/815c81b1055a43e6aeb1495ce75bf348, entries=150, sequenceid=182, filesize=11.9 K 2024-11-16T05:42:19,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for ebfb6a6d32a314c2093cd1145e998024 in 1270ms, sequenceid=182, compaction requested=true 2024-11-16T05:42:19,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:19,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:42:19,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:19,811 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:19,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:42:19,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:19,811 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:19,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:42:19,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:19,811 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:19,811 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/B is initiating minor compaction (all files) 2024-11-16T05:42:19,812 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/B in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:19,812 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e42eabbf4cb14c72902cf4b7f79b91d4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f7b60a1a16a948658b78f8adf8528603, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/0a72f4bc7a1349d7b87ea06ecdaf8c6e] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=35.9 K 2024-11-16T05:42:19,812 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:19,812 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e42eabbf4cb14c72902cf4b7f79b91d4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731735735058 2024-11-16T05:42:19,812 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/A is initiating minor compaction (all files) 2024-11-16T05:42:19,812 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/A in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:19,812 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/1bf831f450d94a5083971c176208f16c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2f64d5cb76064ecbb9a830483b577c37, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/5a9797873f6e4133bad142d246ae6a22] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=108.3 K 2024-11-16T05:42:19,812 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:19,812 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/1bf831f450d94a5083971c176208f16c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2f64d5cb76064ecbb9a830483b577c37, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/5a9797873f6e4133bad142d246ae6a22] 2024-11-16T05:42:19,812 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f7b60a1a16a948658b78f8adf8528603, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731735735710 2024-11-16T05:42:19,813 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bf831f450d94a5083971c176208f16c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731735735058 2024-11-16T05:42:19,813 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a72f4bc7a1349d7b87ea06ecdaf8c6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731735737907 2024-11-16T05:42:19,813 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f64d5cb76064ecbb9a830483b577c37, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731735735710 2024-11-16T05:42:19,813 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a9797873f6e4133bad142d246ae6a22, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731735737907 2024-11-16T05:42:19,817 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:19,818 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:19,819 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/457ab35675164c1987b184a8751363ce is 50, key is test_row_0/B:col10/1731735737908/Put/seqid=0 2024-11-16T05:42:19,819 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411168df73e72627740c7b57a2acae20c078d_ebfb6a6d32a314c2093cd1145e998024 store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:19,821 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411168df73e72627740c7b57a2acae20c078d_ebfb6a6d32a314c2093cd1145e998024, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:19,822 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168df73e72627740c7b57a2acae20c078d_ebfb6a6d32a314c2093cd1145e998024 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:19,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742417_1593 (size=12561) 2024-11-16T05:42:19,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742418_1594 (size=4469) 2024-11-16T05:42:20,229 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/457ab35675164c1987b184a8751363ce as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/457ab35675164c1987b184a8751363ce 2024-11-16T05:42:20,241 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/B of ebfb6a6d32a314c2093cd1145e998024 into 457ab35675164c1987b184a8751363ce(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:20,242 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:20,242 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/B, priority=13, startTime=1731735739811; duration=0sec 2024-11-16T05:42:20,242 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:20,242 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:B 2024-11-16T05:42:20,242 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:20,242 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:20,242 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/6d5d3cb2d1f84cd3a39dbeb0b8fc3c4f is 175, key is test_row_0/A:col10/1731735737908/Put/seqid=0 2024-11-16T05:42:20,243 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:20,243 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/C is initiating minor compaction (all files) 2024-11-16T05:42:20,243 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/C in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:20,243 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/12fff360a8924aca83c11ce66b3825a4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/208e74ac408d4bdb9f0b2464dd6364ef, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/815c81b1055a43e6aeb1495ce75bf348] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=35.9 K 2024-11-16T05:42:20,244 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 12fff360a8924aca83c11ce66b3825a4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731735735058 2024-11-16T05:42:20,244 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 208e74ac408d4bdb9f0b2464dd6364ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731735735710 2024-11-16T05:42:20,244 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 815c81b1055a43e6aeb1495ce75bf348, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731735737907 2024-11-16T05:42:20,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742419_1595 (size=31515) 2024-11-16T05:42:20,258 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/6d5d3cb2d1f84cd3a39dbeb0b8fc3c4f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/6d5d3cb2d1f84cd3a39dbeb0b8fc3c4f 2024-11-16T05:42:20,266 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/A of ebfb6a6d32a314c2093cd1145e998024 into 6d5d3cb2d1f84cd3a39dbeb0b8fc3c4f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:20,266 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:20,266 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/A, priority=13, startTime=1731735739811; duration=0sec 2024-11-16T05:42:20,266 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:20,266 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:A 2024-11-16T05:42:20,267 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#C#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:20,267 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/5febe605aac74e3e86c1a9b031c9e654 is 50, key is test_row_0/C:col10/1731735737908/Put/seqid=0 2024-11-16T05:42:20,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742420_1596 (size=12561) 2024-11-16T05:42:20,278 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/5febe605aac74e3e86c1a9b031c9e654 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/5febe605aac74e3e86c1a9b031c9e654 2024-11-16T05:42:20,286 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/C of ebfb6a6d32a314c2093cd1145e998024 into 5febe605aac74e3e86c1a9b031c9e654(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:20,286 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:20,286 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/C, priority=13, startTime=1731735739811; duration=0sec 2024-11-16T05:42:20,286 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:20,286 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:C 2024-11-16T05:42:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T05:42:20,318 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:20,318 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:42:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-16T05:42:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T05:42:20,320 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:42:20,320 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:42:20,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:42:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T05:42:20,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T05:42:20,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:20,473 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T05:42:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116465dd0749c6f4f279345fce6ffa9ce55_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735738560/Put/seqid=0 2024-11-16T05:42:20,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742421_1597 (size=12304) 2024-11-16T05:42:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T05:42:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:20,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:20,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735800738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:20,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735800741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:20,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735800741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:20,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735800848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735800848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735800848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:20,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:20,890 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116465dd0749c6f4f279345fce6ffa9ce55_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116465dd0749c6f4f279345fce6ffa9ce55_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:20,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/0a4d1d0684c44d5dbeca2de9a677028d, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/0a4d1d0684c44d5dbeca2de9a677028d is 175, key is test_row_0/A:col10/1731735738560/Put/seqid=0 2024-11-16T05:42:20,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742422_1598 (size=31105) 2024-11-16T05:42:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T05:42:21,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735801058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735801058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735801058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,299 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/0a4d1d0684c44d5dbeca2de9a677028d 2024-11-16T05:42:21,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/07a007a82fd1439e81e2d429eed91f8d is 50, key is test_row_0/B:col10/1731735738560/Put/seqid=0 2024-11-16T05:42:21,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742423_1599 (size=12151) 2024-11-16T05:42:21,313 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/07a007a82fd1439e81e2d429eed91f8d 2024-11-16T05:42:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/79911086a9444705bec726872170dca6 is 50, key is test_row_0/C:col10/1731735738560/Put/seqid=0 2024-11-16T05:42:21,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742424_1600 (size=12151) 2024-11-16T05:42:21,335 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/79911086a9444705bec726872170dca6 2024-11-16T05:42:21,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/0a4d1d0684c44d5dbeca2de9a677028d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/0a4d1d0684c44d5dbeca2de9a677028d 2024-11-16T05:42:21,344 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/0a4d1d0684c44d5dbeca2de9a677028d, entries=150, sequenceid=198, filesize=30.4 K 2024-11-16T05:42:21,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/07a007a82fd1439e81e2d429eed91f8d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/07a007a82fd1439e81e2d429eed91f8d 2024-11-16T05:42:21,350 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/07a007a82fd1439e81e2d429eed91f8d, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T05:42:21,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/79911086a9444705bec726872170dca6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/79911086a9444705bec726872170dca6 2024-11-16T05:42:21,360 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/79911086a9444705bec726872170dca6, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T05:42:21,361 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ebfb6a6d32a314c2093cd1145e998024 in 888ms, sequenceid=198, compaction requested=false 2024-11-16T05:42:21,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:21,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:21,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-16T05:42:21,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-16T05:42:21,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-16T05:42:21,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0420 sec 2024-11-16T05:42:21,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.0490 sec 2024-11-16T05:42:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:21,371 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:42:21,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T05:42:21,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111634662647cfa44051a0fda410904adeca_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735740730/Put/seqid=0 2024-11-16T05:42:21,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735801386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735801387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742425_1601 (size=14794) 2024-11-16T05:42:21,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735801393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T05:42:21,458 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:21,459 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:42:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-16T05:42:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T05:42:21,460 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:42:21,461 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:42:21,461 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:42:21,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735801498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735801498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735801508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T05:42:21,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T05:42:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:21,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:21,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735801708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735801708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735801718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:21,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T05:42:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:21,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T05:42:21,793 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:21,796 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111634662647cfa44051a0fda410904adeca_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111634662647cfa44051a0fda410904adeca_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:21,796 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/ce0baf79586242c2b612776b7e190d60, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:21,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/ce0baf79586242c2b612776b7e190d60 is 175, key is test_row_0/A:col10/1731735740730/Put/seqid=0 2024-11-16T05:42:21,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742426_1602 (size=39749) 2024-11-16T05:42:21,800 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/ce0baf79586242c2b612776b7e190d60 2024-11-16T05:42:21,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/c51d5a74f0cb4dd3860d0efcbab48a69 is 50, key is test_row_0/B:col10/1731735740730/Put/seqid=0 2024-11-16T05:42:21,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742427_1603 (size=12151) 2024-11-16T05:42:21,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T05:42:21,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:21,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:21,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:21,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:21,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735802018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:22,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735802018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:22,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735802028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:22,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:22,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T05:42:22,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:22,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T05:42:22,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/c51d5a74f0cb4dd3860d0efcbab48a69 2024-11-16T05:42:22,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/f0d7ee369e9f4574a5d617a9d55e963e is 50, key is test_row_0/C:col10/1731735740730/Put/seqid=0 2024-11-16T05:42:22,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742428_1604 (size=12151) 2024-11-16T05:42:22,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T05:42:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T05:42:22,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:22,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T05:42:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735802528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735802528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:22,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735802538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T05:42:22,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/f0d7ee369e9f4574a5d617a9d55e963e 2024-11-16T05:42:22,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/ce0baf79586242c2b612776b7e190d60 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/ce0baf79586242c2b612776b7e190d60 2024-11-16T05:42:22,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/ce0baf79586242c2b612776b7e190d60, entries=200, sequenceid=223, filesize=38.8 K 2024-11-16T05:42:22,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/c51d5a74f0cb4dd3860d0efcbab48a69 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c51d5a74f0cb4dd3860d0efcbab48a69 2024-11-16T05:42:22,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c51d5a74f0cb4dd3860d0efcbab48a69, entries=150, sequenceid=223, filesize=11.9 K 2024-11-16T05:42:22,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/f0d7ee369e9f4574a5d617a9d55e963e as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/f0d7ee369e9f4574a5d617a9d55e963e 2024-11-16T05:42:22,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/f0d7ee369e9f4574a5d617a9d55e963e, entries=150, sequenceid=223, filesize=11.9 K 2024-11-16T05:42:22,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ebfb6a6d32a314c2093cd1145e998024 in 1261ms, sequenceid=223, compaction requested=true 2024-11-16T05:42:22,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:22,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:42:22,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:22,632 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:22,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:42:22,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:22,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:42:22,632 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:22,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:22,633 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:22,633 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:22,633 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/A is initiating minor compaction (all files) 2024-11-16T05:42:22,633 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/B is initiating minor compaction (all files) 2024-11-16T05:42:22,633 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/B in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,633 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/A in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,633 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/457ab35675164c1987b184a8751363ce, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/07a007a82fd1439e81e2d429eed91f8d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c51d5a74f0cb4dd3860d0efcbab48a69] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=36.0 K 2024-11-16T05:42:22,633 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/6d5d3cb2d1f84cd3a39dbeb0b8fc3c4f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/0a4d1d0684c44d5dbeca2de9a677028d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/ce0baf79586242c2b612776b7e190d60] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=100.0 K 2024-11-16T05:42:22,633 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,633 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/6d5d3cb2d1f84cd3a39dbeb0b8fc3c4f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/0a4d1d0684c44d5dbeca2de9a677028d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/ce0baf79586242c2b612776b7e190d60] 2024-11-16T05:42:22,633 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 457ab35675164c1987b184a8751363ce, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731735737907 2024-11-16T05:42:22,633 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d5d3cb2d1f84cd3a39dbeb0b8fc3c4f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731735737907 2024-11-16T05:42:22,633 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 07a007a82fd1439e81e2d429eed91f8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731735738555 2024-11-16T05:42:22,633 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a4d1d0684c44d5dbeca2de9a677028d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731735738555 2024-11-16T05:42:22,633 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting c51d5a74f0cb4dd3860d0efcbab48a69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731735740730 2024-11-16T05:42:22,634 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce0baf79586242c2b612776b7e190d60, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731735740730 2024-11-16T05:42:22,638 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:22,639 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116b787683a54ed4e05a6931bc41098d4ac_ebfb6a6d32a314c2093cd1145e998024 store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:22,639 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:22,640 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/89811138e47c4caa834cb056e1c78691 is 50, key is test_row_0/B:col10/1731735740730/Put/seqid=0 2024-11-16T05:42:22,641 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116b787683a54ed4e05a6931bc41098d4ac_ebfb6a6d32a314c2093cd1145e998024, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:22,642 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b787683a54ed4e05a6931bc41098d4ac_ebfb6a6d32a314c2093cd1145e998024 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:22,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742429_1605 (size=12663) 2024-11-16T05:42:22,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742430_1606 (size=4469) 2024-11-16T05:42:22,645 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#A#compaction#510 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:22,646 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/b279ec2d96c54be98f71a22cd44a9a7a is 175, key is test_row_0/A:col10/1731735740730/Put/seqid=0 2024-11-16T05:42:22,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742431_1607 (size=31617) 2024-11-16T05:42:22,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T05:42:22,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:22,682 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:42:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167f21fb06811f472e8e6810aca201e8ad_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735741383/Put/seqid=0 2024-11-16T05:42:22,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742432_1608 (size=12304) 2024-11-16T05:42:23,047 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/89811138e47c4caa834cb056e1c78691 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/89811138e47c4caa834cb056e1c78691 2024-11-16T05:42:23,052 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/B of ebfb6a6d32a314c2093cd1145e998024 into 89811138e47c4caa834cb056e1c78691(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:23,052 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:23,052 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/B, priority=13, startTime=1731735742632; duration=0sec 2024-11-16T05:42:23,052 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:23,052 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:B 2024-11-16T05:42:23,052 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:23,052 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/b279ec2d96c54be98f71a22cd44a9a7a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b279ec2d96c54be98f71a22cd44a9a7a 2024-11-16T05:42:23,052 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:23,053 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/C is initiating minor compaction (all files) 2024-11-16T05:42:23,053 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/C in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:23,053 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/5febe605aac74e3e86c1a9b031c9e654, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/79911086a9444705bec726872170dca6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/f0d7ee369e9f4574a5d617a9d55e963e] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=36.0 K 2024-11-16T05:42:23,053 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5febe605aac74e3e86c1a9b031c9e654, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731735737907 2024-11-16T05:42:23,053 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 79911086a9444705bec726872170dca6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731735738555 2024-11-16T05:42:23,053 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f0d7ee369e9f4574a5d617a9d55e963e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731735740730 2024-11-16T05:42:23,056 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/A of ebfb6a6d32a314c2093cd1145e998024 into b279ec2d96c54be98f71a22cd44a9a7a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:23,056 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:23,056 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/A, priority=13, startTime=1731735742632; duration=0sec 2024-11-16T05:42:23,056 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:23,056 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:A 2024-11-16T05:42:23,061 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#C#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:23,061 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/720974f3c6b24433b5a42b89034da79c is 50, key is test_row_0/C:col10/1731735740730/Put/seqid=0 2024-11-16T05:42:23,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742433_1609 (size=12663) 2024-11-16T05:42:23,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,094 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167f21fb06811f472e8e6810aca201e8ad_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167f21fb06811f472e8e6810aca201e8ad_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:23,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/7366f9eb286f4004983cbdfe9287a43f, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/7366f9eb286f4004983cbdfe9287a43f is 175, key is test_row_0/A:col10/1731735741383/Put/seqid=0 2024-11-16T05:42:23,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742434_1610 (size=31105) 2024-11-16T05:42:23,127 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/7366f9eb286f4004983cbdfe9287a43f 2024-11-16T05:42:23,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/f794f9cdb3be4a7fb3514704f6f49f9a is 50, key is test_row_0/B:col10/1731735741383/Put/seqid=0 2024-11-16T05:42:23,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742435_1611 (size=12151) 2024-11-16T05:42:23,148 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/f794f9cdb3be4a7fb3514704f6f49f9a 2024-11-16T05:42:23,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/24eb2823a92748b4b7a1eb05c299128a is 50, key is test_row_0/C:col10/1731735741383/Put/seqid=0 2024-11-16T05:42:23,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742436_1612 (size=12151) 2024-11-16T05:42:23,162 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/24eb2823a92748b4b7a1eb05c299128a 2024-11-16T05:42:23,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/7366f9eb286f4004983cbdfe9287a43f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/7366f9eb286f4004983cbdfe9287a43f 2024-11-16T05:42:23,171 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/7366f9eb286f4004983cbdfe9287a43f, entries=150, sequenceid=234, filesize=30.4 K 2024-11-16T05:42:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/f794f9cdb3be4a7fb3514704f6f49f9a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f794f9cdb3be4a7fb3514704f6f49f9a 2024-11-16T05:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,174 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f794f9cdb3be4a7fb3514704f6f49f9a, entries=150, sequenceid=234, filesize=11.9 K 2024-11-16T05:42:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/24eb2823a92748b4b7a1eb05c299128a as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/24eb2823a92748b4b7a1eb05c299128a 2024-11-16T05:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,178 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/24eb2823a92748b4b7a1eb05c299128a, entries=150, sequenceid=234, filesize=11.9 K 2024-11-16T05:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,179 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for ebfb6a6d32a314c2093cd1145e998024 in 497ms, sequenceid=234, compaction requested=false 2024-11-16T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:23,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:23,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-16T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-16T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-16T05:42:23,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7190 sec 2024-11-16T05:42:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.7240 sec 2024-11-16T05:42:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,470 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/720974f3c6b24433b5a42b89034da79c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/720974f3c6b24433b5a42b89034da79c 2024-11-16T05:42:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,476 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/C of ebfb6a6d32a314c2093cd1145e998024 into 720974f3c6b24433b5a42b89034da79c(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:23,476 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:23,476 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/C, priority=13, startTime=1731735742632; duration=0sec 2024-11-16T05:42:23,476 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:23,476 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:C 2024-11-16T05:42:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,596 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:42:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T05:42:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:23,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164f7b95cb98fa4e16a85140183a1e91f4_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735743575/Put/seqid=0 2024-11-16T05:42:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T05:42:23,608 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:23,608 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:42:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-16T05:42:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T05:42:23,612 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:42:23,612 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:42:23,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:42:23,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742437_1613 (size=14794) 2024-11-16T05:42:23,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735803629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:23,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735803630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:23,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735803633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T05:42:23,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735803738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:23,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735803738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:23,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735803738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:23,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:23,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:23,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:23,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T05:42:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735803948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735803948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735803948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:24,020 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:24,023 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164f7b95cb98fa4e16a85140183a1e91f4_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164f7b95cb98fa4e16a85140183a1e91f4_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:24,024 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/a3f36b15c1f541c2bddb866636077673, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:24,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/a3f36b15c1f541c2bddb866636077673 is 175, key is test_row_0/A:col10/1731735743575/Put/seqid=0 2024-11-16T05:42:24,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742438_1614 (size=39745) 2024-11-16T05:42:24,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:24,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:24,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:24,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T05:42:24,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735804258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:24,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735804259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735804259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:24,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:24,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:24,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,428 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/a3f36b15c1f541c2bddb866636077673 2024-11-16T05:42:24,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/e996b16e72954165985520b975299a0c is 50, key is test_row_0/B:col10/1731735743575/Put/seqid=0 2024-11-16T05:42:24,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742439_1615 (size=9757) 2024-11-16T05:42:24,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:24,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:24,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:24,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:24,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T05:42:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735804768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:24,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735804768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735804768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:24,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:24,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:24,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/e996b16e72954165985520b975299a0c 2024-11-16T05:42:24,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/2e4dd1c88fde4ac3ae06facc6ce87461 is 50, key is test_row_0/C:col10/1731735743575/Put/seqid=0 2024-11-16T05:42:24,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742440_1616 (size=9757) 2024-11-16T05:42:24,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:24,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:24,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:24,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:25,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:25,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:42:25,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/2e4dd1c88fde4ac3ae06facc6ce87461 2024-11-16T05:42:25,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/a3f36b15c1f541c2bddb866636077673 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a3f36b15c1f541c2bddb866636077673 2024-11-16T05:42:25,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a3f36b15c1f541c2bddb866636077673, entries=200, sequenceid=249, filesize=38.8 K 2024-11-16T05:42:25,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/e996b16e72954165985520b975299a0c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e996b16e72954165985520b975299a0c 2024-11-16T05:42:25,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e996b16e72954165985520b975299a0c, entries=100, sequenceid=249, filesize=9.5 K 2024-11-16T05:42:25,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/2e4dd1c88fde4ac3ae06facc6ce87461 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/2e4dd1c88fde4ac3ae06facc6ce87461 2024-11-16T05:42:25,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/2e4dd1c88fde4ac3ae06facc6ce87461, entries=100, sequenceid=249, filesize=9.5 K 2024-11-16T05:42:25,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ebfb6a6d32a314c2093cd1145e998024 in 1670ms, sequenceid=249, compaction requested=true 2024-11-16T05:42:25,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:25,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:42:25,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:25,266 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:25,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:42:25,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:25,266 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:25,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:42:25,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:25,267 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:25,267 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:25,267 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/A is initiating minor compaction (all files) 2024-11-16T05:42:25,267 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/B is initiating minor compaction (all files) 2024-11-16T05:42:25,267 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/B in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:25,267 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/A in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:25,267 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b279ec2d96c54be98f71a22cd44a9a7a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/7366f9eb286f4004983cbdfe9287a43f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a3f36b15c1f541c2bddb866636077673] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=100.1 K 2024-11-16T05:42:25,267 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/89811138e47c4caa834cb056e1c78691, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f794f9cdb3be4a7fb3514704f6f49f9a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e996b16e72954165985520b975299a0c] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=33.8 K 2024-11-16T05:42:25,267 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:25,267 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b279ec2d96c54be98f71a22cd44a9a7a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/7366f9eb286f4004983cbdfe9287a43f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a3f36b15c1f541c2bddb866636077673] 2024-11-16T05:42:25,267 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 89811138e47c4caa834cb056e1c78691, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731735740730 2024-11-16T05:42:25,267 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b279ec2d96c54be98f71a22cd44a9a7a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731735740730 2024-11-16T05:42:25,268 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting f794f9cdb3be4a7fb3514704f6f49f9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731735741381 2024-11-16T05:42:25,268 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7366f9eb286f4004983cbdfe9287a43f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731735741381 2024-11-16T05:42:25,268 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting e996b16e72954165985520b975299a0c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731735743575 2024-11-16T05:42:25,268 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3f36b15c1f541c2bddb866636077673, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731735743575 2024-11-16T05:42:25,271 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:25,273 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#B#compaction#519 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:25,273 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e52ff58160b94710b0ee341c632e8855_ebfb6a6d32a314c2093cd1145e998024 store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:25,273 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/9026d07b3f66442bb20c4f325df6ee4c is 50, key is test_row_0/B:col10/1731735743575/Put/seqid=0 2024-11-16T05:42:25,275 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e52ff58160b94710b0ee341c632e8855_ebfb6a6d32a314c2093cd1145e998024, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:25,275 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e52ff58160b94710b0ee341c632e8855_ebfb6a6d32a314c2093cd1145e998024 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:25,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742441_1617 (size=12765) 2024-11-16T05:42:25,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742442_1618 (size=4469) 2024-11-16T05:42:25,280 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#A#compaction#518 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:25,281 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/68468f3f7fe34b7d8d123c8a48bb7775 is 175, key is test_row_0/A:col10/1731735743575/Put/seqid=0 2024-11-16T05:42:25,283 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/9026d07b3f66442bb20c4f325df6ee4c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/9026d07b3f66442bb20c4f325df6ee4c 2024-11-16T05:42:25,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742443_1619 (size=31826) 2024-11-16T05:42:25,288 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/68468f3f7fe34b7d8d123c8a48bb7775 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/68468f3f7fe34b7d8d123c8a48bb7775 2024-11-16T05:42:25,288 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/B of ebfb6a6d32a314c2093cd1145e998024 into 9026d07b3f66442bb20c4f325df6ee4c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:25,289 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:25,289 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/B, priority=13, startTime=1731735745266; duration=0sec 2024-11-16T05:42:25,289 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:25,289 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:B 2024-11-16T05:42:25,289 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:25,289 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:25,289 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/C is initiating minor compaction (all files) 2024-11-16T05:42:25,290 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/C in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:25,290 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/720974f3c6b24433b5a42b89034da79c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/24eb2823a92748b4b7a1eb05c299128a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/2e4dd1c88fde4ac3ae06facc6ce87461] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=33.8 K 2024-11-16T05:42:25,290 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 720974f3c6b24433b5a42b89034da79c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731735740730 2024-11-16T05:42:25,290 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 24eb2823a92748b4b7a1eb05c299128a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731735741381 2024-11-16T05:42:25,290 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e4dd1c88fde4ac3ae06facc6ce87461, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731735743575 2024-11-16T05:42:25,292 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/A of ebfb6a6d32a314c2093cd1145e998024 into 68468f3f7fe34b7d8d123c8a48bb7775(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:25,293 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:25,293 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/A, priority=13, startTime=1731735745266; duration=0sec 2024-11-16T05:42:25,293 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:25,293 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:A 2024-11-16T05:42:25,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T05:42:25,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:25,295 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T05:42:25,296 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#C#compaction#520 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:25,297 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/9ba1973fd70d4956a88ac14aff6bab71 is 50, key is test_row_0/C:col10/1731735743575/Put/seqid=0 2024-11-16T05:42:25,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166d7bb736217241db96bb26fbba62e4eb_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735743631/Put/seqid=0 2024-11-16T05:42:25,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742444_1620 (size=12765) 2024-11-16T05:42:25,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742445_1621 (size=12454) 2024-11-16T05:42:25,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:25,706 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/9ba1973fd70d4956a88ac14aff6bab71 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9ba1973fd70d4956a88ac14aff6bab71 2024-11-16T05:42:25,706 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166d7bb736217241db96bb26fbba62e4eb_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166d7bb736217241db96bb26fbba62e4eb_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:25,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/2202ff36d6234341bfe72c2b0b9293f9, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:25,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/2202ff36d6234341bfe72c2b0b9293f9 is 175, key is test_row_0/A:col10/1731735743631/Put/seqid=0 2024-11-16T05:42:25,710 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/C of ebfb6a6d32a314c2093cd1145e998024 into 9ba1973fd70d4956a88ac14aff6bab71(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:25,710 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:25,710 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/C, priority=13, startTime=1731735745266; duration=0sec 2024-11-16T05:42:25,710 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:25,710 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:C 2024-11-16T05:42:25,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742446_1622 (size=31255) 2024-11-16T05:42:25,712 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/2202ff36d6234341bfe72c2b0b9293f9 2024-11-16T05:42:25,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/76e3520f1eb041a69f00f74bf5f6b310 is 50, key is test_row_0/B:col10/1731735743631/Put/seqid=0 2024-11-16T05:42:25,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742447_1623 (size=12301) 2024-11-16T05:42:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T05:42:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. as already flushing 2024-11-16T05:42:25,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:25,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735805801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735805802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:25,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735805802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:25,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735805908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:25,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735805908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735805908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:25,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40624 deadline: 1731735805978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:25,981 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_2' in ebfb6a6d32a314c2093cd1145e998024 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:42:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40626 deadline: 1731735806048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,051 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,35555,1731735575219 for 'test_row_1' in ebfb6a6d32a314c2093cd1145e998024 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18318 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:42:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735806118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735806118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735806118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,135 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/76e3520f1eb041a69f00f74bf5f6b310 2024-11-16T05:42:26,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/53d81aa5b6474611a8cdfed461df4e99 is 50, key is test_row_0/C:col10/1731735743631/Put/seqid=0 2024-11-16T05:42:26,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742448_1624 (size=12301) 2024-11-16T05:42:26,152 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/53d81aa5b6474611a8cdfed461df4e99 2024-11-16T05:42:26,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/2202ff36d6234341bfe72c2b0b9293f9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2202ff36d6234341bfe72c2b0b9293f9 2024-11-16T05:42:26,159 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2202ff36d6234341bfe72c2b0b9293f9, entries=150, sequenceid=275, filesize=30.5 K 2024-11-16T05:42:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/76e3520f1eb041a69f00f74bf5f6b310 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/76e3520f1eb041a69f00f74bf5f6b310 2024-11-16T05:42:26,165 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/76e3520f1eb041a69f00f74bf5f6b310, entries=150, sequenceid=275, filesize=12.0 K 2024-11-16T05:42:26,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/53d81aa5b6474611a8cdfed461df4e99 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/53d81aa5b6474611a8cdfed461df4e99 2024-11-16T05:42:26,170 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/53d81aa5b6474611a8cdfed461df4e99, entries=150, sequenceid=275, filesize=12.0 K 2024-11-16T05:42:26,171 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ebfb6a6d32a314c2093cd1145e998024 in 876ms, sequenceid=275, compaction requested=false 2024-11-16T05:42:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-16T05:42:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-16T05:42:26,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-16T05:42:26,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5600 sec 2024-11-16T05:42:26,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.5660 sec 2024-11-16T05:42:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:26,431 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:42:26,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T05:42:26,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111631502bca5687432f9c5f5dbc40b712f4_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735746430/Put/seqid=0 2024-11-16T05:42:26,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742449_1625 (size=12454) 2024-11-16T05:42:26,469 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:26,472 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111631502bca5687432f9c5f5dbc40b712f4_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111631502bca5687432f9c5f5dbc40b712f4_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:26,472 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/91fd02bbed6e4cd49d41151d3ee2d1c5, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:26,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/91fd02bbed6e4cd49d41151d3ee2d1c5 is 175, key is test_row_0/A:col10/1731735746430/Put/seqid=0 2024-11-16T05:42:26,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735806468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742450_1626 (size=31255) 2024-11-16T05:42:26,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735806472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735806474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735806578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735806588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735806588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735806788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735806798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735806798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:26,876 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/91fd02bbed6e4cd49d41151d3ee2d1c5 2024-11-16T05:42:26,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/4abd68ea67e9459f866dd62d3e309f0b is 50, key is test_row_0/B:col10/1731735746430/Put/seqid=0 2024-11-16T05:42:26,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742451_1627 (size=12301) 2024-11-16T05:42:26,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/4abd68ea67e9459f866dd62d3e309f0b 2024-11-16T05:42:26,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/88ccb702670d4e2da89a61d2fa275138 is 50, key is test_row_0/C:col10/1731735746430/Put/seqid=0 2024-11-16T05:42:26,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742452_1628 (size=12301) 2024-11-16T05:42:27,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735807098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735807108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735807108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/88ccb702670d4e2da89a61d2fa275138 2024-11-16T05:42:27,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/91fd02bbed6e4cd49d41151d3ee2d1c5 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/91fd02bbed6e4cd49d41151d3ee2d1c5 2024-11-16T05:42:27,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/91fd02bbed6e4cd49d41151d3ee2d1c5, entries=150, sequenceid=291, filesize=30.5 K 2024-11-16T05:42:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/4abd68ea67e9459f866dd62d3e309f0b as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4abd68ea67e9459f866dd62d3e309f0b 2024-11-16T05:42:27,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4abd68ea67e9459f866dd62d3e309f0b, entries=150, sequenceid=291, filesize=12.0 K 2024-11-16T05:42:27,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/88ccb702670d4e2da89a61d2fa275138 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/88ccb702670d4e2da89a61d2fa275138 2024-11-16T05:42:27,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/88ccb702670d4e2da89a61d2fa275138, entries=150, sequenceid=291, filesize=12.0 K 2024-11-16T05:42:27,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ebfb6a6d32a314c2093cd1145e998024 in 889ms, sequenceid=291, compaction requested=true 2024-11-16T05:42:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:27,320 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:42:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:42:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:27,320 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:42:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:27,321 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:27,321 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/A is initiating minor compaction (all files) 2024-11-16T05:42:27,321 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/A in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:27,321 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/68468f3f7fe34b7d8d123c8a48bb7775, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2202ff36d6234341bfe72c2b0b9293f9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/91fd02bbed6e4cd49d41151d3ee2d1c5] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=92.1 K 2024-11-16T05:42:27,321 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:27,321 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/68468f3f7fe34b7d8d123c8a48bb7775, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2202ff36d6234341bfe72c2b0b9293f9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/91fd02bbed6e4cd49d41151d3ee2d1c5] 2024-11-16T05:42:27,321 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:27,321 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/B is initiating minor compaction (all files) 2024-11-16T05:42:27,321 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/B in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:27,322 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/9026d07b3f66442bb20c4f325df6ee4c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/76e3520f1eb041a69f00f74bf5f6b310, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4abd68ea67e9459f866dd62d3e309f0b] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=36.5 K 2024-11-16T05:42:27,322 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68468f3f7fe34b7d8d123c8a48bb7775, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731735741390 2024-11-16T05:42:27,322 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9026d07b3f66442bb20c4f325df6ee4c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731735741390 2024-11-16T05:42:27,322 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2202ff36d6234341bfe72c2b0b9293f9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731735743626 2024-11-16T05:42:27,322 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 76e3520f1eb041a69f00f74bf5f6b310, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731735743626 2024-11-16T05:42:27,322 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91fd02bbed6e4cd49d41151d3ee2d1c5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731735745800 2024-11-16T05:42:27,323 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4abd68ea67e9459f866dd62d3e309f0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731735745800 2024-11-16T05:42:27,327 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:27,328 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#B#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:27,329 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/10c1c7b693864fc3bdcaa2301559efa4 is 50, key is test_row_0/B:col10/1731735746430/Put/seqid=0 2024-11-16T05:42:27,330 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411161324c1dda5144b11bb77308a53eb9f06_ebfb6a6d32a314c2093cd1145e998024 store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:27,332 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411161324c1dda5144b11bb77308a53eb9f06_ebfb6a6d32a314c2093cd1145e998024, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:27,332 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161324c1dda5144b11bb77308a53eb9f06_ebfb6a6d32a314c2093cd1145e998024 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:27,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742453_1629 (size=13017) 2024-11-16T05:42:27,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742454_1630 (size=4469) 2024-11-16T05:42:27,340 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#A#compaction#528 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:27,341 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/a73a09d542cc47c280cc692d38e70984 is 175, key is test_row_0/A:col10/1731735746430/Put/seqid=0 2024-11-16T05:42:27,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742455_1631 (size=31971) 2024-11-16T05:42:27,354 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/a73a09d542cc47c280cc692d38e70984 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a73a09d542cc47c280cc692d38e70984 2024-11-16T05:42:27,359 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/A of ebfb6a6d32a314c2093cd1145e998024 into a73a09d542cc47c280cc692d38e70984(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:27,359 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:27,359 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/A, priority=13, startTime=1731735747320; duration=0sec 2024-11-16T05:42:27,359 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:27,359 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:A 2024-11-16T05:42:27,359 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:27,374 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:27,374 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/C is initiating minor compaction (all files) 2024-11-16T05:42:27,374 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/C in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:27,374 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9ba1973fd70d4956a88ac14aff6bab71, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/53d81aa5b6474611a8cdfed461df4e99, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/88ccb702670d4e2da89a61d2fa275138] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=36.5 K 2024-11-16T05:42:27,374 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ba1973fd70d4956a88ac14aff6bab71, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731735741390 2024-11-16T05:42:27,375 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53d81aa5b6474611a8cdfed461df4e99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731735743626 2024-11-16T05:42:27,375 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88ccb702670d4e2da89a61d2fa275138, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731735745800 2024-11-16T05:42:27,383 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#C#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:27,383 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/e193a28ed7894972897bf37f588cb568 is 50, key is test_row_0/C:col10/1731735746430/Put/seqid=0 2024-11-16T05:42:27,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742456_1632 (size=13017) 2024-11-16T05:42:27,408 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/e193a28ed7894972897bf37f588cb568 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/e193a28ed7894972897bf37f588cb568 2024-11-16T05:42:27,414 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/C of ebfb6a6d32a314c2093cd1145e998024 into e193a28ed7894972897bf37f588cb568(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:27,414 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:27,415 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/C, priority=13, startTime=1731735747320; duration=0sec 2024-11-16T05:42:27,415 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:27,415 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:C 2024-11-16T05:42:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:27,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:42:27,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T05:42:27,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160a32e24c63be4590a4cba25a264bf45c_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735746472/Put/seqid=0 2024-11-16T05:42:27,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742457_1633 (size=12454) 2024-11-16T05:42:27,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735807639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735807641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735807641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,698 INFO [Thread-2431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2431. 2024-11-16T05:42:27,698 DEBUG [Thread-2431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:27,698 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:27,698 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:27,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:27,699 INFO [Thread-2433 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2433. 2024-11-16T05:42:27,699 DEBUG [Thread-2433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:27,699 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:27,699 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:27,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:27,701 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-11-16T05:42:27,701 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:27,701 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:27,701 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:27,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:27,702 INFO [Thread-2435 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2435. 2024-11-16T05:42:27,702 INFO [Thread-2437 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2437. 2024-11-16T05:42:27,702 DEBUG [Thread-2437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:27,702 DEBUG [Thread-2435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:27,702 DEBUG [Thread-2435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:27,702 DEBUG [Thread-2437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:27,702 DEBUG [Thread-2437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:27,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:27,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:27,703 DEBUG [Thread-2435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:27,741 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/10c1c7b693864fc3bdcaa2301559efa4 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/10c1c7b693864fc3bdcaa2301559efa4 2024-11-16T05:42:27,744 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/B of ebfb6a6d32a314c2093cd1145e998024 into 10c1c7b693864fc3bdcaa2301559efa4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:27,744 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:27,744 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/B, priority=13, startTime=1731735747320; duration=0sec 2024-11-16T05:42:27,745 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:27,745 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:B 2024-11-16T05:42:27,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735807748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735807748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735807748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T05:42:27,788 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735807959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735807959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735807959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:28,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:28,036 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160a32e24c63be4590a4cba25a264bf45c_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160a32e24c63be4590a4cba25a264bf45c_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:28,037 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/b8022276b6404c918758e355dbf61d25, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:28,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/b8022276b6404c918758e355dbf61d25 is 175, key is test_row_0/A:col10/1731735746472/Put/seqid=0 2024-11-16T05:42:28,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742458_1634 (size=31255) 2024-11-16T05:42:28,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:28,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:28,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735808270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735808269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735808269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:28,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/b8022276b6404c918758e355dbf61d25 2024-11-16T05:42:28,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/dbfd06b039ff4be0b2fc0d9652a24dff is 50, key is test_row_0/B:col10/1731735746472/Put/seqid=0 2024-11-16T05:42:28,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742459_1635 (size=12301) 2024-11-16T05:42:28,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40614 deadline: 1731735808779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:28,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:28,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:42:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40650 deadline: 1731735808779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40638 deadline: 1731735808779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 , the old value is region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:42:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., hostname=3456ee6a3164,35555,1731735575219, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:42:28,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/dbfd06b039ff4be0b2fc0d9652a24dff 2024-11-16T05:42:28,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/625618b9ad7a48209085202ca085a9a3 is 50, key is test_row_0/C:col10/1731735746472/Put/seqid=0 2024-11-16T05:42:28,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742460_1636 (size=12301) 2024-11-16T05:42:29,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/625618b9ad7a48209085202ca085a9a3 2024-11-16T05:42:29,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/b8022276b6404c918758e355dbf61d25 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b8022276b6404c918758e355dbf61d25 2024-11-16T05:42:29,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b8022276b6404c918758e355dbf61d25, entries=150, sequenceid=315, filesize=30.5 K 2024-11-16T05:42:29,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/dbfd06b039ff4be0b2fc0d9652a24dff as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dbfd06b039ff4be0b2fc0d9652a24dff 2024-11-16T05:42:29,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dbfd06b039ff4be0b2fc0d9652a24dff, entries=150, sequenceid=315, filesize=12.0 K 2024-11-16T05:42:29,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/625618b9ad7a48209085202ca085a9a3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/625618b9ad7a48209085202ca085a9a3 2024-11-16T05:42:29,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/625618b9ad7a48209085202ca085a9a3, entries=150, sequenceid=315, filesize=12.0 K 2024-11-16T05:42:29,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for ebfb6a6d32a314c2093cd1145e998024 in 1671ms, sequenceid=315, compaction requested=false 2024-11-16T05:42:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35555 {}] regionserver.HRegion(8855): Flush requested on ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:29,790 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T05:42:29,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T05:42:29,791 INFO [Thread-2418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2418. 2024-11-16T05:42:29,791 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-11-16T05:42:29,791 DEBUG [Thread-2418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:29,791 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:29,791 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:29,791 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:29,791 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:29,791 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:29,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:29,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:29,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e52fea83f3e34e4382643264e0884556_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735749788/Put/seqid=0 2024-11-16T05:42:29,800 INFO [Thread-2424 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2424. 2024-11-16T05:42:29,800 DEBUG [Thread-2424 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:29,800 DEBUG [Thread-2424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:29,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742461_1637 (size=12454) 2024-11-16T05:42:29,801 DEBUG [Thread-2424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:29,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:30,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:30,203 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e52fea83f3e34e4382643264e0884556_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e52fea83f3e34e4382643264e0884556_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:30,204 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/4ea09c130fdd4aeba10a301c6b55ada3, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:30,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/4ea09c130fdd4aeba10a301c6b55ada3 is 175, key is test_row_0/A:col10/1731735749788/Put/seqid=0 2024-11-16T05:42:30,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742462_1638 (size=31255) 2024-11-16T05:42:30,608 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/4ea09c130fdd4aeba10a301c6b55ada3 2024-11-16T05:42:30,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/84f7ef0bb8e3463d86183ba1df1b7c74 is 50, key is test_row_0/B:col10/1731735749788/Put/seqid=0 2024-11-16T05:42:30,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742463_1639 (size=12301) 2024-11-16T05:42:31,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/84f7ef0bb8e3463d86183ba1df1b7c74 2024-11-16T05:42:31,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/ca1146fc75134db48bf47af2685650c9 is 50, key is test_row_0/C:col10/1731735749788/Put/seqid=0 2024-11-16T05:42:31,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742464_1640 (size=12301) 2024-11-16T05:42:31,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/ca1146fc75134db48bf47af2685650c9 2024-11-16T05:42:31,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/4ea09c130fdd4aeba10a301c6b55ada3 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/4ea09c130fdd4aeba10a301c6b55ada3 2024-11-16T05:42:31,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/4ea09c130fdd4aeba10a301c6b55ada3, entries=150, sequenceid=333, filesize=30.5 K 2024-11-16T05:42:31,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/84f7ef0bb8e3463d86183ba1df1b7c74 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/84f7ef0bb8e3463d86183ba1df1b7c74 2024-11-16T05:42:31,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/84f7ef0bb8e3463d86183ba1df1b7c74, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T05:42:31,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/ca1146fc75134db48bf47af2685650c9 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/ca1146fc75134db48bf47af2685650c9 2024-11-16T05:42:31,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/ca1146fc75134db48bf47af2685650c9, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T05:42:31,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=6.71 KB/6870 for ebfb6a6d32a314c2093cd1145e998024 in 1259ms, sequenceid=333, compaction requested=true 2024-11-16T05:42:31,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:31,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:42:31,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:31,049 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:31,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:42:31,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:31,050 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:31,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebfb6a6d32a314c2093cd1145e998024:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:42:31,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:31,050 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:31,050 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:31,050 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/A is initiating minor compaction (all files) 2024-11-16T05:42:31,050 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/B is initiating minor compaction (all files) 2024-11-16T05:42:31,050 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/B in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:31,050 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/A in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:31,050 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/10c1c7b693864fc3bdcaa2301559efa4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dbfd06b039ff4be0b2fc0d9652a24dff, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/84f7ef0bb8e3463d86183ba1df1b7c74] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=36.7 K 2024-11-16T05:42:31,050 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a73a09d542cc47c280cc692d38e70984, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b8022276b6404c918758e355dbf61d25, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/4ea09c130fdd4aeba10a301c6b55ada3] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=92.3 K 2024-11-16T05:42:31,050 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:31,050 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. files: [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a73a09d542cc47c280cc692d38e70984, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b8022276b6404c918758e355dbf61d25, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/4ea09c130fdd4aeba10a301c6b55ada3] 2024-11-16T05:42:31,051 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10c1c7b693864fc3bdcaa2301559efa4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731735745800 2024-11-16T05:42:31,051 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting a73a09d542cc47c280cc692d38e70984, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731735745800 2024-11-16T05:42:31,051 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbfd06b039ff4be0b2fc0d9652a24dff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731735746465 2024-11-16T05:42:31,051 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting b8022276b6404c918758e355dbf61d25, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731735746465 2024-11-16T05:42:31,051 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84f7ef0bb8e3463d86183ba1df1b7c74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731735747638 2024-11-16T05:42:31,051 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea09c130fdd4aeba10a301c6b55ada3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731735747638 2024-11-16T05:42:31,056 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:31,057 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#B#compaction#536 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:31,057 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116eac2c326410a4a499188d36fda137d22_ebfb6a6d32a314c2093cd1145e998024 store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:31,057 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/8d0fd94f23d941999eaa6e262a771959 is 50, key is test_row_0/B:col10/1731735749788/Put/seqid=0 2024-11-16T05:42:31,059 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116eac2c326410a4a499188d36fda137d22_ebfb6a6d32a314c2093cd1145e998024, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:31,059 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eac2c326410a4a499188d36fda137d22_ebfb6a6d32a314c2093cd1145e998024 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:31,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742465_1641 (size=13119) 2024-11-16T05:42:31,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742466_1642 (size=4469) 2024-11-16T05:42:31,465 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/8d0fd94f23d941999eaa6e262a771959 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/8d0fd94f23d941999eaa6e262a771959 2024-11-16T05:42:31,467 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#A#compaction#537 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:31,468 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/308c5e06edaa4ce4a092c7ed83cfbc19 is 175, key is test_row_0/A:col10/1731735749788/Put/seqid=0 2024-11-16T05:42:31,471 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/B of ebfb6a6d32a314c2093cd1145e998024 into 8d0fd94f23d941999eaa6e262a771959(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:31,471 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:31,471 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/B, priority=13, startTime=1731735751049; duration=0sec 2024-11-16T05:42:31,471 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:42:31,471 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:B 2024-11-16T05:42:31,471 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:42:31,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742467_1643 (size=32073) 2024-11-16T05:42:31,472 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:42:31,472 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1541): ebfb6a6d32a314c2093cd1145e998024/C is initiating minor compaction (all files) 2024-11-16T05:42:31,472 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebfb6a6d32a314c2093cd1145e998024/C in TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:31,472 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/e193a28ed7894972897bf37f588cb568, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/625618b9ad7a48209085202ca085a9a3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/ca1146fc75134db48bf47af2685650c9] into tmpdir=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp, totalSize=36.7 K 2024-11-16T05:42:31,473 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e193a28ed7894972897bf37f588cb568, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731735745800 2024-11-16T05:42:31,473 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 625618b9ad7a48209085202ca085a9a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731735746465 2024-11-16T05:42:31,474 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca1146fc75134db48bf47af2685650c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731735747638 2024-11-16T05:42:31,482 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebfb6a6d32a314c2093cd1145e998024#C#compaction#538 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:42:31,482 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/a145d081d3d4497b96452dc899e84ed6 is 50, key is test_row_0/C:col10/1731735749788/Put/seqid=0 2024-11-16T05:42:31,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742468_1644 (size=13119) 2024-11-16T05:42:31,875 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/308c5e06edaa4ce4a092c7ed83cfbc19 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/308c5e06edaa4ce4a092c7ed83cfbc19 2024-11-16T05:42:31,879 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/A of ebfb6a6d32a314c2093cd1145e998024 into 308c5e06edaa4ce4a092c7ed83cfbc19(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:31,879 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:31,879 INFO [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/A, priority=13, startTime=1731735751049; duration=0sec 2024-11-16T05:42:31,879 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:31,879 DEBUG [RS:0;3456ee6a3164:35555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:A 2024-11-16T05:42:31,890 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/a145d081d3d4497b96452dc899e84ed6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/a145d081d3d4497b96452dc899e84ed6 2024-11-16T05:42:31,893 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebfb6a6d32a314c2093cd1145e998024/C of ebfb6a6d32a314c2093cd1145e998024 into a145d081d3d4497b96452dc899e84ed6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:42:31,894 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebfb6a6d32a314c2093cd1145e998024: 2024-11-16T05:42:31,894 INFO [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., storeName=ebfb6a6d32a314c2093cd1145e998024/C, priority=13, startTime=1731735751050; duration=0sec 2024-11-16T05:42:31,894 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:42:31,894 DEBUG [RS:0;3456ee6a3164:35555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebfb6a6d32a314c2093cd1145e998024:C 2024-11-16T05:42:33,556 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:42:36,032 INFO [Thread-2426 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2426. 2024-11-16T05:42:36,032 DEBUG [Thread-2426 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:36,033 DEBUG [Thread-2426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:36,033 DEBUG [Thread-2426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:36,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:36,161 INFO [Thread-2422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2422. 2024-11-16T05:42:36,162 DEBUG [Thread-2422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:42:36,162 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:36,162 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:36,162 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:36,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3975 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3918 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3894 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3985 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3947 2024-11-16T05:42:36,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:42:36,163 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:42:36,163 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:42:36,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:36,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:36,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:36,165 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:42:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T05:42:36,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735756171"}]},"ts":"1731735756171"} 2024-11-16T05:42:36,173 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:42:36,173 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:42:36,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:42:36,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, UNASSIGN}] 2024-11-16T05:42:36,176 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, UNASSIGN 2024-11-16T05:42:36,177 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=ebfb6a6d32a314c2093cd1145e998024, regionState=CLOSING, regionLocation=3456ee6a3164,35555,1731735575219 2024-11-16T05:42:36,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, UNASSIGN because future has completed 2024-11-16T05:42:36,179 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:42:36,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219}] 2024-11-16T05:42:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T05:42:36,332 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:36,332 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:42:36,332 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing ebfb6a6d32a314c2093cd1145e998024, disabling compactions & flushes 2024-11-16T05:42:36,332 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:36,332 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:36,332 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. after waiting 0 ms 2024-11-16T05:42:36,333 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:36,333 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing ebfb6a6d32a314c2093cd1145e998024 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-16T05:42:36,344 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111609fef9c90273455ba20cea34a985c556_ebfb6a6d32a314c2093cd1145e998024 is 50, key is test_row_0/A:col10/1731735749799/Put/seqid=0 2024-11-16T05:42:36,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742469_1645 (size=12454) 2024-11-16T05:42:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T05:42:36,748 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:42:36,752 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111609fef9c90273455ba20cea34a985c556_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111609fef9c90273455ba20cea34a985c556_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:36,753 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/9fe4876fb4da4bf6934ee88076bea89c, store: [table=TestAcidGuarantees family=A region=ebfb6a6d32a314c2093cd1145e998024] 2024-11-16T05:42:36,753 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/9fe4876fb4da4bf6934ee88076bea89c is 175, key is test_row_0/A:col10/1731735749799/Put/seqid=0 2024-11-16T05:42:36,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742470_1646 (size=31255) 2024-11-16T05:42:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T05:42:37,157 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/9fe4876fb4da4bf6934ee88076bea89c 2024-11-16T05:42:37,161 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/a21473c5774f4e97801f9de14994764f is 50, key is test_row_0/B:col10/1731735749799/Put/seqid=0 2024-11-16T05:42:37,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742471_1647 (size=12301) 2024-11-16T05:42:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T05:42:37,564 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/a21473c5774f4e97801f9de14994764f 2024-11-16T05:42:37,569 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/06e1a2890379461badc8f8e9ff3573e6 is 50, key is test_row_0/C:col10/1731735749799/Put/seqid=0 2024-11-16T05:42:37,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742472_1648 (size=12301) 2024-11-16T05:42:37,973 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/06e1a2890379461badc8f8e9ff3573e6 2024-11-16T05:42:37,976 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/A/9fe4876fb4da4bf6934ee88076bea89c as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/9fe4876fb4da4bf6934ee88076bea89c 2024-11-16T05:42:37,979 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/9fe4876fb4da4bf6934ee88076bea89c, entries=150, sequenceid=342, filesize=30.5 K 2024-11-16T05:42:37,980 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/B/a21473c5774f4e97801f9de14994764f as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/a21473c5774f4e97801f9de14994764f 2024-11-16T05:42:37,983 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/a21473c5774f4e97801f9de14994764f, entries=150, sequenceid=342, filesize=12.0 K 2024-11-16T05:42:37,983 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/.tmp/C/06e1a2890379461badc8f8e9ff3573e6 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/06e1a2890379461badc8f8e9ff3573e6 2024-11-16T05:42:37,986 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/06e1a2890379461badc8f8e9ff3573e6, entries=150, sequenceid=342, filesize=12.0 K 2024-11-16T05:42:37,986 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for ebfb6a6d32a314c2093cd1145e998024 in 1653ms, sequenceid=342, compaction requested=false 2024-11-16T05:42:37,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/972c62cf5b854c6f9fece53eaef1e529, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b99b23db4fc9444e9fe13e2912852271, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/e4a08cfa56a24b11b55a26f7dda6472e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/99eb4514e0d8469ab10e879cd11242cb, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/55add32ccc1f4210bca7937ac9cef55a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/54b8cd6b81b246cda5cab4bb2a7ac304, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/01082936c37a406bb5a9216d5a1c6066, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/3f90395abb4146529a2703ad13ccb6c6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/c1b60b6aa8654442ac260c0769994121, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/1bf831f450d94a5083971c176208f16c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2f64d5cb76064ecbb9a830483b577c37, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/5a9797873f6e4133bad142d246ae6a22, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/6d5d3cb2d1f84cd3a39dbeb0b8fc3c4f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/0a4d1d0684c44d5dbeca2de9a677028d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/ce0baf79586242c2b612776b7e190d60, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b279ec2d96c54be98f71a22cd44a9a7a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/7366f9eb286f4004983cbdfe9287a43f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a3f36b15c1f541c2bddb866636077673, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/68468f3f7fe34b7d8d123c8a48bb7775, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2202ff36d6234341bfe72c2b0b9293f9, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a73a09d542cc47c280cc692d38e70984, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/91fd02bbed6e4cd49d41151d3ee2d1c5, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b8022276b6404c918758e355dbf61d25, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/4ea09c130fdd4aeba10a301c6b55ada3] to archive 2024-11-16T05:42:37,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:42:37,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/972c62cf5b854c6f9fece53eaef1e529 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/972c62cf5b854c6f9fece53eaef1e529 2024-11-16T05:42:37,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b99b23db4fc9444e9fe13e2912852271 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b99b23db4fc9444e9fe13e2912852271 2024-11-16T05:42:37,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/e4a08cfa56a24b11b55a26f7dda6472e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/e4a08cfa56a24b11b55a26f7dda6472e 2024-11-16T05:42:37,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/99eb4514e0d8469ab10e879cd11242cb to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/99eb4514e0d8469ab10e879cd11242cb 2024-11-16T05:42:37,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/55add32ccc1f4210bca7937ac9cef55a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/55add32ccc1f4210bca7937ac9cef55a 2024-11-16T05:42:37,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/54b8cd6b81b246cda5cab4bb2a7ac304 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/54b8cd6b81b246cda5cab4bb2a7ac304 2024-11-16T05:42:37,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/01082936c37a406bb5a9216d5a1c6066 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/01082936c37a406bb5a9216d5a1c6066 2024-11-16T05:42:37,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/3f90395abb4146529a2703ad13ccb6c6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/3f90395abb4146529a2703ad13ccb6c6 2024-11-16T05:42:37,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/c1b60b6aa8654442ac260c0769994121 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/c1b60b6aa8654442ac260c0769994121 2024-11-16T05:42:37,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/1bf831f450d94a5083971c176208f16c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/1bf831f450d94a5083971c176208f16c 2024-11-16T05:42:37,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2f64d5cb76064ecbb9a830483b577c37 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2f64d5cb76064ecbb9a830483b577c37 2024-11-16T05:42:37,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/5a9797873f6e4133bad142d246ae6a22 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/5a9797873f6e4133bad142d246ae6a22 2024-11-16T05:42:38,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/6d5d3cb2d1f84cd3a39dbeb0b8fc3c4f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/6d5d3cb2d1f84cd3a39dbeb0b8fc3c4f 2024-11-16T05:42:38,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/0a4d1d0684c44d5dbeca2de9a677028d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/0a4d1d0684c44d5dbeca2de9a677028d 2024-11-16T05:42:38,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/ce0baf79586242c2b612776b7e190d60 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/ce0baf79586242c2b612776b7e190d60 2024-11-16T05:42:38,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b279ec2d96c54be98f71a22cd44a9a7a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b279ec2d96c54be98f71a22cd44a9a7a 2024-11-16T05:42:38,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/7366f9eb286f4004983cbdfe9287a43f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/7366f9eb286f4004983cbdfe9287a43f 2024-11-16T05:42:38,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a3f36b15c1f541c2bddb866636077673 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a3f36b15c1f541c2bddb866636077673 2024-11-16T05:42:38,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/68468f3f7fe34b7d8d123c8a48bb7775 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/68468f3f7fe34b7d8d123c8a48bb7775 2024-11-16T05:42:38,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2202ff36d6234341bfe72c2b0b9293f9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/2202ff36d6234341bfe72c2b0b9293f9 2024-11-16T05:42:38,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a73a09d542cc47c280cc692d38e70984 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/a73a09d542cc47c280cc692d38e70984 2024-11-16T05:42:38,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/91fd02bbed6e4cd49d41151d3ee2d1c5 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/91fd02bbed6e4cd49d41151d3ee2d1c5 2024-11-16T05:42:38,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b8022276b6404c918758e355dbf61d25 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/b8022276b6404c918758e355dbf61d25 2024-11-16T05:42:38,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/4ea09c130fdd4aeba10a301c6b55ada3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/4ea09c130fdd4aeba10a301c6b55ada3 2024-11-16T05:42:38,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/14301cc129d8463ca4962ccfb4862d3b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dcfe3666547244ad821a3dd7a8bdcf38, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e882c9ab684a4c86bb2d2134124c8d03, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c0a5a80bb93045b989c0e240eec3ac08, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/d7de74cf05004debb27ad7c1e903510b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f94ee81ef7b9438aa5e1e355f7964cc8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/20b603f699d14d8f941ae933452d276b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4819f5b610ae4c7f9213d353a3dd573a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e42eabbf4cb14c72902cf4b7f79b91d4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/b733099e52994abc8b4b7364e49af095, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f7b60a1a16a948658b78f8adf8528603, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/457ab35675164c1987b184a8751363ce, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/0a72f4bc7a1349d7b87ea06ecdaf8c6e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/07a007a82fd1439e81e2d429eed91f8d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/89811138e47c4caa834cb056e1c78691, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c51d5a74f0cb4dd3860d0efcbab48a69, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f794f9cdb3be4a7fb3514704f6f49f9a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/9026d07b3f66442bb20c4f325df6ee4c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e996b16e72954165985520b975299a0c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/76e3520f1eb041a69f00f74bf5f6b310, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/10c1c7b693864fc3bdcaa2301559efa4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4abd68ea67e9459f866dd62d3e309f0b, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dbfd06b039ff4be0b2fc0d9652a24dff, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/84f7ef0bb8e3463d86183ba1df1b7c74] to archive 2024-11-16T05:42:38,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:42:38,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/14301cc129d8463ca4962ccfb4862d3b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/14301cc129d8463ca4962ccfb4862d3b 2024-11-16T05:42:38,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dcfe3666547244ad821a3dd7a8bdcf38 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dcfe3666547244ad821a3dd7a8bdcf38 2024-11-16T05:42:38,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e882c9ab684a4c86bb2d2134124c8d03 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e882c9ab684a4c86bb2d2134124c8d03 2024-11-16T05:42:38,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c0a5a80bb93045b989c0e240eec3ac08 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c0a5a80bb93045b989c0e240eec3ac08 2024-11-16T05:42:38,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/d7de74cf05004debb27ad7c1e903510b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/d7de74cf05004debb27ad7c1e903510b 2024-11-16T05:42:38,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f94ee81ef7b9438aa5e1e355f7964cc8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f94ee81ef7b9438aa5e1e355f7964cc8 2024-11-16T05:42:38,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/20b603f699d14d8f941ae933452d276b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/20b603f699d14d8f941ae933452d276b 2024-11-16T05:42:38,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4819f5b610ae4c7f9213d353a3dd573a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4819f5b610ae4c7f9213d353a3dd573a 2024-11-16T05:42:38,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e42eabbf4cb14c72902cf4b7f79b91d4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e42eabbf4cb14c72902cf4b7f79b91d4 2024-11-16T05:42:38,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/b733099e52994abc8b4b7364e49af095 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/b733099e52994abc8b4b7364e49af095 2024-11-16T05:42:38,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f7b60a1a16a948658b78f8adf8528603 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f7b60a1a16a948658b78f8adf8528603 2024-11-16T05:42:38,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/457ab35675164c1987b184a8751363ce to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/457ab35675164c1987b184a8751363ce 2024-11-16T05:42:38,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/0a72f4bc7a1349d7b87ea06ecdaf8c6e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/0a72f4bc7a1349d7b87ea06ecdaf8c6e 2024-11-16T05:42:38,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/07a007a82fd1439e81e2d429eed91f8d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/07a007a82fd1439e81e2d429eed91f8d 2024-11-16T05:42:38,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/89811138e47c4caa834cb056e1c78691 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/89811138e47c4caa834cb056e1c78691 2024-11-16T05:42:38,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c51d5a74f0cb4dd3860d0efcbab48a69 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/c51d5a74f0cb4dd3860d0efcbab48a69 2024-11-16T05:42:38,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f794f9cdb3be4a7fb3514704f6f49f9a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/f794f9cdb3be4a7fb3514704f6f49f9a 2024-11-16T05:42:38,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/9026d07b3f66442bb20c4f325df6ee4c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/9026d07b3f66442bb20c4f325df6ee4c 2024-11-16T05:42:38,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e996b16e72954165985520b975299a0c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/e996b16e72954165985520b975299a0c 2024-11-16T05:42:38,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/76e3520f1eb041a69f00f74bf5f6b310 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/76e3520f1eb041a69f00f74bf5f6b310 2024-11-16T05:42:38,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/10c1c7b693864fc3bdcaa2301559efa4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/10c1c7b693864fc3bdcaa2301559efa4 2024-11-16T05:42:38,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4abd68ea67e9459f866dd62d3e309f0b to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/4abd68ea67e9459f866dd62d3e309f0b 2024-11-16T05:42:38,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dbfd06b039ff4be0b2fc0d9652a24dff to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/dbfd06b039ff4be0b2fc0d9652a24dff 2024-11-16T05:42:38,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/84f7ef0bb8e3463d86183ba1df1b7c74 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/84f7ef0bb8e3463d86183ba1df1b7c74 2024-11-16T05:42:38,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/faeb604d617944f9be9acb77245e077d, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/36dddfb4443d4c51a8bad92c90452fc8, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/de9f5b6234b541ecb0aff73e8bd70c5f, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/83ee8f6005fa403c8d59efe975f504c6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/003d0af8163f4f4d8c4b208c7b785751, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/e5016c0229944aedbd23081c3efff437, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/b054c854af5e40fd894d2d305f51b4c7, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9b2c1836443340da9584203a1db1796c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/12fff360a8924aca83c11ce66b3825a4, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/6e819b3883a44109bb3a88e6503d0599, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/208e74ac408d4bdb9f0b2464dd6364ef, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/5febe605aac74e3e86c1a9b031c9e654, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/815c81b1055a43e6aeb1495ce75bf348, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/79911086a9444705bec726872170dca6, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/720974f3c6b24433b5a42b89034da79c, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/f0d7ee369e9f4574a5d617a9d55e963e, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/24eb2823a92748b4b7a1eb05c299128a, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9ba1973fd70d4956a88ac14aff6bab71, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/2e4dd1c88fde4ac3ae06facc6ce87461, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/53d81aa5b6474611a8cdfed461df4e99, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/e193a28ed7894972897bf37f588cb568, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/88ccb702670d4e2da89a61d2fa275138, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/625618b9ad7a48209085202ca085a9a3, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/ca1146fc75134db48bf47af2685650c9] to archive 2024-11-16T05:42:38,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:42:38,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/faeb604d617944f9be9acb77245e077d to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/faeb604d617944f9be9acb77245e077d 2024-11-16T05:42:38,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/36dddfb4443d4c51a8bad92c90452fc8 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/36dddfb4443d4c51a8bad92c90452fc8 2024-11-16T05:42:38,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/de9f5b6234b541ecb0aff73e8bd70c5f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/de9f5b6234b541ecb0aff73e8bd70c5f 2024-11-16T05:42:38,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/83ee8f6005fa403c8d59efe975f504c6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/83ee8f6005fa403c8d59efe975f504c6 2024-11-16T05:42:38,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/003d0af8163f4f4d8c4b208c7b785751 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/003d0af8163f4f4d8c4b208c7b785751 2024-11-16T05:42:38,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/e5016c0229944aedbd23081c3efff437 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/e5016c0229944aedbd23081c3efff437 2024-11-16T05:42:38,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/b054c854af5e40fd894d2d305f51b4c7 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/b054c854af5e40fd894d2d305f51b4c7 2024-11-16T05:42:38,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9b2c1836443340da9584203a1db1796c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9b2c1836443340da9584203a1db1796c 2024-11-16T05:42:38,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/12fff360a8924aca83c11ce66b3825a4 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/12fff360a8924aca83c11ce66b3825a4 2024-11-16T05:42:38,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/6e819b3883a44109bb3a88e6503d0599 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/6e819b3883a44109bb3a88e6503d0599 2024-11-16T05:42:38,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/208e74ac408d4bdb9f0b2464dd6364ef to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/208e74ac408d4bdb9f0b2464dd6364ef 2024-11-16T05:42:38,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/5febe605aac74e3e86c1a9b031c9e654 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/5febe605aac74e3e86c1a9b031c9e654 2024-11-16T05:42:38,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/815c81b1055a43e6aeb1495ce75bf348 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/815c81b1055a43e6aeb1495ce75bf348 2024-11-16T05:42:38,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/79911086a9444705bec726872170dca6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/79911086a9444705bec726872170dca6 2024-11-16T05:42:38,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/720974f3c6b24433b5a42b89034da79c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/720974f3c6b24433b5a42b89034da79c 2024-11-16T05:42:38,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/f0d7ee369e9f4574a5d617a9d55e963e to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/f0d7ee369e9f4574a5d617a9d55e963e 2024-11-16T05:42:38,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/24eb2823a92748b4b7a1eb05c299128a to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/24eb2823a92748b4b7a1eb05c299128a 2024-11-16T05:42:38,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9ba1973fd70d4956a88ac14aff6bab71 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/9ba1973fd70d4956a88ac14aff6bab71 2024-11-16T05:42:38,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/2e4dd1c88fde4ac3ae06facc6ce87461 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/2e4dd1c88fde4ac3ae06facc6ce87461 2024-11-16T05:42:38,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/53d81aa5b6474611a8cdfed461df4e99 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/53d81aa5b6474611a8cdfed461df4e99 2024-11-16T05:42:38,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/e193a28ed7894972897bf37f588cb568 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/e193a28ed7894972897bf37f588cb568 2024-11-16T05:42:38,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/88ccb702670d4e2da89a61d2fa275138 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/88ccb702670d4e2da89a61d2fa275138 2024-11-16T05:42:38,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/625618b9ad7a48209085202ca085a9a3 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/625618b9ad7a48209085202ca085a9a3 2024-11-16T05:42:38,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/ca1146fc75134db48bf47af2685650c9 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/ca1146fc75134db48bf47af2685650c9 2024-11-16T05:42:38,048 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/recovered.edits/345.seqid, newMaxSeqId=345, maxSeqId=4 2024-11-16T05:42:38,049 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. 2024-11-16T05:42:38,049 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for ebfb6a6d32a314c2093cd1145e998024: Waiting for close lock at 1731735756332Running coprocessor pre-close hooks at 1731735756332Disabling compacts and flushes for region at 1731735756332Disabling writes for close at 1731735756333 (+1 ms)Obtaining lock to block concurrent updates at 1731735756333Preparing flush snapshotting stores in ebfb6a6d32a314c2093cd1145e998024 at 1731735756333Finished memstore snapshotting TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731735756333Flushing stores of TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024. at 1731735756334 (+1 ms)Flushing ebfb6a6d32a314c2093cd1145e998024/A: creating writer at 1731735756334Flushing mob file ebfb6a6d32a314c2093cd1145e998024/A: appending metadata at 1731735756344 (+10 ms)Flushing mob file ebfb6a6d32a314c2093cd1145e998024/A: closing flushed file at 1731735756344Flushing ebfb6a6d32a314c2093cd1145e998024/A: appending metadata at 1731735756753 (+409 ms)Flushing ebfb6a6d32a314c2093cd1145e998024/A: closing flushed file at 1731735756753Flushing ebfb6a6d32a314c2093cd1145e998024/B: creating writer at 1731735757159 (+406 ms)Flushing ebfb6a6d32a314c2093cd1145e998024/B: appending metadata at 1731735757160 (+1 ms)Flushing ebfb6a6d32a314c2093cd1145e998024/B: closing flushed file at 1731735757160Flushing ebfb6a6d32a314c2093cd1145e998024/C: creating writer at 1731735757567 (+407 ms)Flushing ebfb6a6d32a314c2093cd1145e998024/C: appending metadata at 1731735757569 (+2 ms)Flushing ebfb6a6d32a314c2093cd1145e998024/C: closing flushed file at 1731735757569Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46f0787d: reopening flushed file at 1731735757976 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d96079d: reopening flushed file at 1731735757979 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@151ee40b: reopening flushed file at 1731735757983 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for ebfb6a6d32a314c2093cd1145e998024 in 1653ms, sequenceid=342, compaction requested=false at 1731735757986 (+3 ms)Writing region close event to WAL at 1731735758046 (+60 ms)Running coprocessor post-close hooks at 1731735758049 (+3 ms)Closed at 1731735758049 2024-11-16T05:42:38,051 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,051 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=ebfb6a6d32a314c2093cd1145e998024, regionState=CLOSED 2024-11-16T05:42:38,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 because future has completed 2024-11-16T05:42:38,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-16T05:42:38,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure ebfb6a6d32a314c2093cd1145e998024, server=3456ee6a3164,35555,1731735575219 in 1.8740 sec 2024-11-16T05:42:38,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-16T05:42:38,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebfb6a6d32a314c2093cd1145e998024, UNASSIGN in 1.8800 sec 2024-11-16T05:42:38,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-16T05:42:38,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8820 sec 2024-11-16T05:42:38,059 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731735758059"}]},"ts":"1731735758059"} 2024-11-16T05:42:38,060 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:42:38,060 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:42:38,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8960 sec 2024-11-16T05:42:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T05:42:38,318 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:38,319 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:42:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:38,320 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T05:42:38,321 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:38,322 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,324 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C, FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/recovered.edits] 2024-11-16T05:42:38,326 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/308c5e06edaa4ce4a092c7ed83cfbc19 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/308c5e06edaa4ce4a092c7ed83cfbc19 2024-11-16T05:42:38,327 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/9fe4876fb4da4bf6934ee88076bea89c to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/A/9fe4876fb4da4bf6934ee88076bea89c 2024-11-16T05:42:38,328 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/8d0fd94f23d941999eaa6e262a771959 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/8d0fd94f23d941999eaa6e262a771959 2024-11-16T05:42:38,329 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/a21473c5774f4e97801f9de14994764f to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/B/a21473c5774f4e97801f9de14994764f 2024-11-16T05:42:38,331 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/06e1a2890379461badc8f8e9ff3573e6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/06e1a2890379461badc8f8e9ff3573e6 2024-11-16T05:42:38,332 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/a145d081d3d4497b96452dc899e84ed6 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/C/a145d081d3d4497b96452dc899e84ed6 2024-11-16T05:42:38,333 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/recovered.edits/345.seqid to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024/recovered.edits/345.seqid 2024-11-16T05:42:38,334 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/default/TestAcidGuarantees/ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,334 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:42:38,334 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:42:38,335 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T05:42:38,336 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116067dfb0de82845c483c1434e4390f2d9_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116067dfb0de82845c483c1434e4390f2d9_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,337 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111609fef9c90273455ba20cea34a985c556_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111609fef9c90273455ba20cea34a985c556_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,338 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160a32e24c63be4590a4cba25a264bf45c_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160a32e24c63be4590a4cba25a264bf45c_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,338 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111631502bca5687432f9c5f5dbc40b712f4_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111631502bca5687432f9c5f5dbc40b712f4_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,339 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111634662647cfa44051a0fda410904adeca_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111634662647cfa44051a0fda410904adeca_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,340 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116465dd0749c6f4f279345fce6ffa9ce55_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116465dd0749c6f4f279345fce6ffa9ce55_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,340 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164c734373f8d749099b5e6649f47150a2_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164c734373f8d749099b5e6649f47150a2_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,341 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164f7b95cb98fa4e16a85140183a1e91f4_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164f7b95cb98fa4e16a85140183a1e91f4_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,342 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c1b8d491e4742efad7cf258760e5fec_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c1b8d491e4742efad7cf258760e5fec_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,343 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116658e82881814499799114d81ecd357ff_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116658e82881814499799114d81ecd357ff_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,344 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166d7bb736217241db96bb26fbba62e4eb_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166d7bb736217241db96bb26fbba62e4eb_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,345 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167f21fb06811f472e8e6810aca201e8ad_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167f21fb06811f472e8e6810aca201e8ad_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,346 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a43825c66c03455582286aeccf06020a_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a43825c66c03455582286aeccf06020a_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,347 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a65584b00fce49eeb6ad9ec63f29c97c_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a65584b00fce49eeb6ad9ec63f29c97c_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,348 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d4dd71a43d4f42d1b496ca7fd46e67d9_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d4dd71a43d4f42d1b496ca7fd46e67d9_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,349 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da2afdec4fb2438b83e81352ff4b24ff_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da2afdec4fb2438b83e81352ff4b24ff_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,350 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e52fea83f3e34e4382643264e0884556_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e52fea83f3e34e4382643264e0884556_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,351 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fd32beb2bf354e79b896a385e921f209_ebfb6a6d32a314c2093cd1145e998024 to hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fd32beb2bf354e79b896a385e921f209_ebfb6a6d32a314c2093cd1145e998024 2024-11-16T05:42:38,352 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:42:38,354 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:38,356 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:42:38,358 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:42:38,359 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:38,359 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:42:38,360 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731735758359"}]},"ts":"9223372036854775807"} 2024-11-16T05:42:38,362 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:42:38,362 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ebfb6a6d32a314c2093cd1145e998024, NAME => 'TestAcidGuarantees,,1731735724572.ebfb6a6d32a314c2093cd1145e998024.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:42:38,362 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:42:38,362 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731735758362"}]},"ts":"9223372036854775807"} 2024-11-16T05:42:38,364 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:42:38,365 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:42:38,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-16T05:42:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T05:42:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:42:38,430 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:42:38,442 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 237), OpenFileDescriptor=451 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=344 (was 387), ProcessCount=11 (was 11), AvailableMemoryMB=3379 (was 4065) 2024-11-16T05:42:38,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T05:42:38,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:42:38,442 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:42:38,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:38,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:38,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:42:38,443 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T05:42:38,443 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1284221299, stopped=false 2024-11-16T05:42:38,443 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3456ee6a3164,39409,1731735574536 2024-11-16T05:42:38,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:42:38,445 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:42:38,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:42:38,445 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:42:38,445 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:42:38,445 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:42:38,446 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:42:38,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:38,446 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:42:38,446 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:42:38,446 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,35555,1731735575219' ***** 2024-11-16T05:42:38,446 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T05:42:38,446 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:42:38,447 INFO [RS:0;3456ee6a3164:35555 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:42:38,447 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:42:38,447 INFO [RS:0;3456ee6a3164:35555 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:42:38,447 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,35555,1731735575219 2024-11-16T05:42:38,447 INFO [RS:0;3456ee6a3164:35555 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:42:38,447 INFO [RS:0;3456ee6a3164:35555 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3456ee6a3164:35555. 2024-11-16T05:42:38,447 DEBUG [RS:0;3456ee6a3164:35555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:42:38,447 DEBUG [RS:0;3456ee6a3164:35555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:38,448 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:42:38,448 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:42:38,448 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:42:38,448 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T05:42:38,448 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T05:42:38,448 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T05:42:38,449 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:42:38,449 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:42:38,449 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:42:38,449 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:42:38,449 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:42:38,449 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:42:38,449 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-16T05:42:38,473 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/info/7cac71c9c1114fd696e716c396b14ead is 95, key is TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb./info:/1731735605355/DeleteFamily/seqid=0 2024-11-16T05:42:38,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742473_1649 (size=6514) 2024-11-16T05:42:38,501 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:42:38,649 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:42:38,849 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:42:38,877 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/info/7cac71c9c1114fd696e716c396b14ead 2024-11-16T05:42:38,904 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/ns/94fd4f32ffbe4d678f301f0d29efa35d is 93, key is TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb./ns:/1731735605314/DeleteFamily/seqid=0 2024-11-16T05:42:38,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742474_1650 (size=5906) 2024-11-16T05:42:39,050 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:42:39,073 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T05:42:39,073 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T05:42:39,250 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:42:39,307 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/ns/94fd4f32ffbe4d678f301f0d29efa35d 2024-11-16T05:42:39,329 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/rep_barrier/6a03a7c3129b438b8cd9b1666ec6be36 is 102, key is TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb./rep_barrier:/1731735605314/DeleteFamily/seqid=0 2024-11-16T05:42:39,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742475_1651 (size=6025) 2024-11-16T05:42:39,450 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:42:39,650 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:42:39,732 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/rep_barrier/6a03a7c3129b438b8cd9b1666ec6be36 2024-11-16T05:42:39,754 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/table/ee8215249643415996f78eb9c333f199 is 96, key is TestAcidGuarantees,,1731735579107.3992ee2db1edacc4ba76f5d2411f31eb./table:/1731735605314/DeleteFamily/seqid=0 2024-11-16T05:42:39,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742476_1652 (size=5936) 2024-11-16T05:42:39,851 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:42:40,051 DEBUG [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:42:40,158 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/table/ee8215249643415996f78eb9c333f199 2024-11-16T05:42:40,161 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/info/7cac71c9c1114fd696e716c396b14ead as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/info/7cac71c9c1114fd696e716c396b14ead 2024-11-16T05:42:40,164 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/info/7cac71c9c1114fd696e716c396b14ead, entries=12, sequenceid=90, filesize=6.4 K 2024-11-16T05:42:40,165 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/ns/94fd4f32ffbe4d678f301f0d29efa35d as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/ns/94fd4f32ffbe4d678f301f0d29efa35d 2024-11-16T05:42:40,167 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/ns/94fd4f32ffbe4d678f301f0d29efa35d, entries=8, sequenceid=90, filesize=5.8 K 2024-11-16T05:42:40,168 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/rep_barrier/6a03a7c3129b438b8cd9b1666ec6be36 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/rep_barrier/6a03a7c3129b438b8cd9b1666ec6be36 2024-11-16T05:42:40,170 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/rep_barrier/6a03a7c3129b438b8cd9b1666ec6be36, entries=6, sequenceid=90, filesize=5.9 K 2024-11-16T05:42:40,171 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/.tmp/table/ee8215249643415996f78eb9c333f199 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/table/ee8215249643415996f78eb9c333f199 2024-11-16T05:42:40,173 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/table/ee8215249643415996f78eb9c333f199, entries=7, sequenceid=90, filesize=5.8 K 2024-11-16T05:42:40,173 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1724ms, sequenceid=90, compaction requested=false 2024-11-16T05:42:40,178 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-16T05:42:40,179 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:42:40,179 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:42:40,179 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731735758448Running coprocessor pre-close hooks at 1731735758449 (+1 ms)Disabling compacts and flushes for region at 1731735758449Disabling writes for close at 1731735758449Obtaining lock to block concurrent updates at 1731735758449Preparing flush snapshotting stores in 1588230740 at 1731735758449Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731735758449Flushing stores of hbase:meta,,1.1588230740 at 1731735758450 (+1 ms)Flushing 1588230740/info: creating writer at 1731735758450Flushing 1588230740/info: appending metadata at 1731735758473 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731735758473Flushing 1588230740/ns: creating writer at 1731735758888 (+415 ms)Flushing 1588230740/ns: appending metadata at 1731735758903 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731735758903Flushing 1588230740/rep_barrier: creating writer at 1731735759311 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731735759328 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731735759328Flushing 1588230740/table: creating writer at 1731735759736 (+408 ms)Flushing 1588230740/table: appending metadata at 1731735759754 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731735759754Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ce4679d: reopening flushed file at 1731735760161 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44d0d30e: reopening flushed file at 1731735760164 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@325209b6: reopening flushed file at 1731735760168 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@457bd837: reopening flushed file at 1731735760170 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1724ms, sequenceid=90, compaction requested=false at 1731735760173 (+3 ms)Writing region close event to WAL at 1731735760176 (+3 ms)Running coprocessor post-close hooks at 1731735760179 (+3 ms)Closed at 1731735760179 2024-11-16T05:42:40,179 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T05:42:40,251 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,35555,1731735575219; all regions closed. 2024-11-16T05:42:40,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741834_1010 (size=25344) 2024-11-16T05:42:40,258 DEBUG [RS:0;3456ee6a3164:35555 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/oldWALs 2024-11-16T05:42:40,258 INFO [RS:0;3456ee6a3164:35555 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C35555%2C1731735575219.meta:.meta(num 1731735578714) 2024-11-16T05:42:40,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741833_1009 (size=14641974) 2024-11-16T05:42:40,262 DEBUG [RS:0;3456ee6a3164:35555 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/oldWALs 2024-11-16T05:42:40,262 INFO [RS:0;3456ee6a3164:35555 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C35555%2C1731735575219:(num 1731735578170) 2024-11-16T05:42:40,262 DEBUG [RS:0;3456ee6a3164:35555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:42:40,262 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:42:40,262 INFO [RS:0;3456ee6a3164:35555 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:42:40,263 INFO [RS:0;3456ee6a3164:35555 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T05:42:40,263 INFO [RS:0;3456ee6a3164:35555 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:42:40,264 INFO [RS:0;3456ee6a3164:35555 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35555 2024-11-16T05:42:40,264 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:42:40,268 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,35555,1731735575219 2024-11-16T05:42:40,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:42:40,269 INFO [RS:0;3456ee6a3164:35555 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:42:40,269 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,35555,1731735575219] 2024-11-16T05:42:40,272 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,35555,1731735575219 already deleted, retry=false 2024-11-16T05:42:40,273 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3456ee6a3164,35555,1731735575219 expired; onlineServers=0 2024-11-16T05:42:40,273 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3456ee6a3164,39409,1731735574536' ***** 2024-11-16T05:42:40,273 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T05:42:40,273 INFO [M:0;3456ee6a3164:39409 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:42:40,273 INFO [M:0;3456ee6a3164:39409 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:42:40,273 DEBUG [M:0;3456ee6a3164:39409 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T05:42:40,273 DEBUG [M:0;3456ee6a3164:39409 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T05:42:40,273 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731735577835 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731735577835,5,FailOnTimeoutGroup] 2024-11-16T05:42:40,273 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731735577836 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731735577836,5,FailOnTimeoutGroup] 2024-11-16T05:42:40,274 INFO [M:0;3456ee6a3164:39409 {}] hbase.ChoreService(370): Chore service for: master/3456ee6a3164:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T05:42:40,273 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T05:42:40,274 INFO [M:0;3456ee6a3164:39409 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:42:40,274 DEBUG [M:0;3456ee6a3164:39409 {}] master.HMaster(1795): Stopping service threads 2024-11-16T05:42:40,274 INFO [M:0;3456ee6a3164:39409 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T05:42:40,274 INFO [M:0;3456ee6a3164:39409 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:42:40,274 ERROR [M:0;3456ee6a3164:39409 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-16T05:42:40,275 INFO [M:0;3456ee6a3164:39409 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:42:40,275 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T05:42:40,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T05:42:40,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:42:40,281 DEBUG [M:0;3456ee6a3164:39409 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-16T05:42:40,281 DEBUG [M:0;3456ee6a3164:39409 {}] master.ActiveMasterManager(353): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-16T05:42:40,281 INFO [M:0;3456ee6a3164:39409 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/.lastflushedseqids 2024-11-16T05:42:40,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742477_1653 (size=72) 2024-11-16T05:42:40,373 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:42:40,373 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35555-0x100470b30e60001, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:42:40,373 INFO [RS:0;3456ee6a3164:35555 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:42:40,373 INFO [RS:0;3456ee6a3164:35555 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,35555,1731735575219; zookeeper connection closed. 2024-11-16T05:42:40,374 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@24043fe9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@24043fe9 2024-11-16T05:42:40,374 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-16T05:42:40,695 INFO [M:0;3456ee6a3164:39409 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:42:40,695 INFO [M:0;3456ee6a3164:39409 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T05:42:40,695 DEBUG [M:0;3456ee6a3164:39409 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:42:40,696 INFO [M:0;3456ee6a3164:39409 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:42:40,696 DEBUG [M:0;3456ee6a3164:39409 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:42:40,696 DEBUG [M:0;3456ee6a3164:39409 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:42:40,696 DEBUG [M:0;3456ee6a3164:39409 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:42:40,696 INFO [M:0;3456ee6a3164:39409 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=737.14 KB heapSize=904.44 KB 2024-11-16T05:42:40,717 DEBUG [M:0;3456ee6a3164:39409 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21dd4c8a0e8944ffa21596437d7872d5 is 82, key is hbase:meta,,1/info:regioninfo/1731735578794/Put/seqid=0 2024-11-16T05:42:40,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742478_1654 (size=5672) 2024-11-16T05:42:41,122 INFO [M:0;3456ee6a3164:39409 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2055 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21dd4c8a0e8944ffa21596437d7872d5 2024-11-16T05:42:41,151 DEBUG [M:0;3456ee6a3164:39409 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/238da28764d34ae4a9e25821ec3d7cab is 2276, key is \x00\x00\x00\x00\x00\x00\x00d/proc:d/1731735663847/Put/seqid=0 2024-11-16T05:42:41,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742479_1655 (size=44886) 2024-11-16T05:42:41,155 INFO [M:0;3456ee6a3164:39409 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=736.58 KB at sequenceid=2055 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/238da28764d34ae4a9e25821ec3d7cab 2024-11-16T05:42:41,159 INFO [M:0;3456ee6a3164:39409 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 238da28764d34ae4a9e25821ec3d7cab 2024-11-16T05:42:41,172 DEBUG [M:0;3456ee6a3164:39409 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f2f2693cd254e6f9e0fbbc2e1b95a09 is 69, key is 3456ee6a3164,35555,1731735575219/rs:state/1731735577905/Put/seqid=0 2024-11-16T05:42:41,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073742480_1656 (size=5156) 2024-11-16T05:42:41,576 INFO [M:0;3456ee6a3164:39409 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2055 (bloomFilter=true), to=hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f2f2693cd254e6f9e0fbbc2e1b95a09 2024-11-16T05:42:41,580 DEBUG [M:0;3456ee6a3164:39409 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21dd4c8a0e8944ffa21596437d7872d5 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/21dd4c8a0e8944ffa21596437d7872d5 2024-11-16T05:42:41,583 INFO [M:0;3456ee6a3164:39409 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/21dd4c8a0e8944ffa21596437d7872d5, entries=8, sequenceid=2055, filesize=5.5 K 2024-11-16T05:42:41,584 DEBUG [M:0;3456ee6a3164:39409 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/238da28764d34ae4a9e25821ec3d7cab as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/238da28764d34ae4a9e25821ec3d7cab 2024-11-16T05:42:41,587 INFO [M:0;3456ee6a3164:39409 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 238da28764d34ae4a9e25821ec3d7cab 2024-11-16T05:42:41,587 INFO [M:0;3456ee6a3164:39409 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/238da28764d34ae4a9e25821ec3d7cab, entries=174, sequenceid=2055, filesize=43.8 K 2024-11-16T05:42:41,588 DEBUG [M:0;3456ee6a3164:39409 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f2f2693cd254e6f9e0fbbc2e1b95a09 as hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7f2f2693cd254e6f9e0fbbc2e1b95a09 2024-11-16T05:42:41,591 INFO [M:0;3456ee6a3164:39409 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35865/user/jenkins/test-data/5dafb4c5-61cf-3bd2-cb80-f08d75cd165d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7f2f2693cd254e6f9e0fbbc2e1b95a09, entries=1, sequenceid=2055, filesize=5.0 K 2024-11-16T05:42:41,591 INFO [M:0;3456ee6a3164:39409 {}] regionserver.HRegion(3140): Finished flush of dataSize ~737.14 KB/754829, heapSize ~904.14 KB/925840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 895ms, sequenceid=2055, compaction requested=false 2024-11-16T05:42:41,593 INFO [M:0;3456ee6a3164:39409 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:42:41,593 DEBUG [M:0;3456ee6a3164:39409 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731735760695Disabling compacts and flushes for region at 1731735760695Disabling writes for close at 1731735760696 (+1 ms)Obtaining lock to block concurrent updates at 1731735760696Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731735760696Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=754829, getHeapSize=926080, getOffHeapSize=0, getCellsCount=2254 at 1731735760697 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731735760698 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731735760698Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731735760717 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731735760717Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731735761132 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731735761151 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731735761151Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731735761159 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731735761172 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731735761172Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@508f38b9: reopening flushed file at 1731735761579 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@170969e1: reopening flushed file at 1731735761583 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@342a80ee: reopening flushed file at 1731735761587 (+4 ms)Finished flush of dataSize ~737.14 KB/754829, heapSize ~904.14 KB/925840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 895ms, sequenceid=2055, compaction requested=false at 1731735761591 (+4 ms)Writing region close event to WAL at 1731735761593 (+2 ms)Closed at 1731735761593 2024-11-16T05:42:41,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33791 is added to blk_1073741830_1006 (size=889619) 2024-11-16T05:42:41,596 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:42:41,596 INFO [M:0;3456ee6a3164:39409 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T05:42:41,597 INFO [M:0;3456ee6a3164:39409 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39409 2024-11-16T05:42:41,597 INFO [M:0;3456ee6a3164:39409 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:42:41,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:42:41,698 INFO [M:0;3456ee6a3164:39409 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:42:41,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39409-0x100470b30e60000, quorum=127.0.0.1:53518, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:42:41,703 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:42:41,704 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T05:42:41,705 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T05:42:41,705 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T05:42:41,705 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/hadoop.log.dir/,STOPPED} 2024-11-16T05:42:41,708 WARN [BP-2113210957-172.17.0.2-1731735571282 heartbeating to localhost/127.0.0.1:35865 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T05:42:41,708 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T05:42:41,708 WARN [BP-2113210957-172.17.0.2-1731735571282 heartbeating to localhost/127.0.0.1:35865 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2113210957-172.17.0.2-1731735571282 (Datanode Uuid 122d7910-34dd-4638-982b-e246163848a6) service to localhost/127.0.0.1:35865 2024-11-16T05:42:41,708 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T05:42:41,710 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/cluster_d224ca95-f133-78db-847d-de6e87cd9c78/data/data1/current/BP-2113210957-172.17.0.2-1731735571282 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T05:42:41,710 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/cluster_d224ca95-f133-78db-847d-de6e87cd9c78/data/data2/current/BP-2113210957-172.17.0.2-1731735571282 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T05:42:41,711 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T05:42:41,718 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T05:42:41,719 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T05:42:41,719 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T05:42:41,719 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T05:42:41,720 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67bf1382-f537-69b8-a71f-3197ef60c466/hadoop.log.dir/,STOPPED} 2024-11-16T05:42:41,737 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T05:42:41,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down